def _write_get(self, cursor, is_list = False):
count = yield cursor.count(with_limit_and_skip=True)
is_list = count > 1 or is_list
if not count:
self.write('[]')
raise tornado.gen.Return(count)
elif is_list:
self.write('[\n')
# Write first entry
yield cursor.fetch_next
resource = cursor.next_object()
resource = yield self._post_get(resource)
json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
while (yield cursor.fetch_next):
self.write(',\n')
resource = cursor.next_object()
resource = yield self._post_get(resource)
json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
if is_list:
self.write('\n]')
raise tornado.gen.Return(count)
def _create_child(self, key, resource):
http_client = AsyncHTTPClient()
url = "{protocol}://{host}{path}?validate=false&complete_links=false"
links = []
query = []
for r in resource[key]:
links.append(r) if "rel" in r and "href" in r else query.append(r)
if query:
response = yield tornado.gen.Task(
http_client.fetch,
url.format(protocol = self.request.protocol, host = self.request.host, path = self.reverse_url(key)),
method = "POST",
body = dumps_mongo(query),
request_timeout = 180,
validate_cert = False,
client_cert = settings.CLIENT_SSL_OPTIONS['certfile'],
client_key = settings.CLIENT_SSL_OPTIONS['keyfile'],
headers = { "Cache-Control": "no-cache",
"Content-Type": MIME["PSJSON"],
"connection": "close" }
)
if response.code >= 400:
raise Exception("Could not add child resource")
response = json.loads(response.body)
if not isinstance(response, list):
links.append({ "href": response["selfRef"], "rel": "full" })
else:
for r in response:
links.append({ "href": r["selfRef"], "rel": "full" })
raise tornado.gen.Return({ "collection": key, "hrefs": links })
def _create_child(self, key, resource):
http_client = AsyncHTTPClient()
url = "{protocol}://{host}{path}?validate=false&complete_links=false"
links = []
query = []
for r in resource[key]:
links.append(r) if "rel" in r and "href" in r else query.append(r)
if query:
response = yield tornado.gen.Task(
http_client.fetch,
url.format(protocol=self.request.protocol,
host=self.request.host,
path=self.reverse_url(key)),
method="POST",
body=dumps_mongo(query),
request_timeout=180,
validate_cert=False,
client_cert=settings.CLIENT_SSL_OPTIONS['certfile'],
client_key=settings.CLIENT_SSL_OPTIONS['keyfile'],
headers={
"Cache-Control": "no-cache",
"Content-Type": MIME["PSJSON"],
"connection": "close"
})
if response.code >= 400:
raise Exception("Could not add child resource")
response = json.loads(response.body)
if not isinstance(response, list):
links.append({"href": response["selfRef"], "rel": "full"})
else:
for r in response:
links.append({"href": r["selfRef"], "rel": "full"})
raise tornado.gen.Return({"collection": key, "hrefs": links})
def _return_resources(self, query):
try:
cursor = self._find(query = query)
response = []
while (yield cursor.fetch_next):
resource = ObjectDict._from_mongo(cursor.next_object())
response.append(resource)
if len(response) == 1:
location = self.request.full_url().split('?')[0]
if not location.endswith(response[0][self.Id]):
location = location + "/" + response[0][self.Id]
self.set_header("Location", location)
self.write(dumps_mongo(response[0], indent=2))
else:
self.write(dumps_mongo(response, indent=2))
except Exception as exp:
raise ValueError(exp)
def _return_resources(self, query):
try:
cursor = self._find(query=query)
response = []
while (yield cursor.fetch_next):
resource = ObjectDict._from_mongo(cursor.next_object())
response.append(resource)
if len(response) == 1:
location = self.request.full_url().split('?')[0]
if not location.endswith(response[0][self.Id]):
location = location + "/" + response[0][self.Id]
self.set_header("Location", location)
self.write(dumps_mongo(response[0], indent=2))
else:
self.write(dumps_mongo(response, indent=2))
except Exception as exp:
raise ValueError(exp)
def _write_get(self, cursor, is_list=False, inline=False):
count = yield cursor.count(with_limit_and_skip=True)
is_list = count != 1 or is_list
if self.accept_content_type == MIME["PSBSON"]:
results = []
while (yield cursor.fetch_next):
resource = yield self._post_get(cursor.next_object(), inline)
results.append(resource)
if not is_list:
results = results[0]
else:
results = {"data": results}
self.write(bson_encode(results))
else:
if not count:
self.write('[]')
raise tornado.gen.Return(count)
elif is_list:
self.write('[\n')
# Write first entry
yield cursor.fetch_next
resource = cursor.next_object()
resource = yield self._post_get(resource, inline)
json_response = dumps_mongo(resource, indent=2).replace(
'\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
while (yield cursor.fetch_next):
self.write(',\n')
resource = cursor.next_object()
resource = yield self._post_get(resource, inline)
json_response = dumps_mongo(resource, indent=2).replace(
'\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
if is_list:
self.write('\n]')
raise tornado.gen.Return(count)
def get(self, res_id=None):
if not res_id:
self.write_error(403,
message="Data request must include a metadata id")
return
# Parse arguments and set up query
try:
parsed = yield self._parse_get_arguments()
options = dict(query=parsed["query"]["query"],
limit=parsed["limit"],
sort=parsed["sort"],
skip=parsed["skip"])
if not options["limit"]:
options.pop("limit", None)
options["query"]["\\$status"] = {"$ne": "DELETED"}
options["fields"] = {"_id": 0}
except Exception as exp:
self.write_error(403, message=exp)
return
# we don't want to wait here since this is a direct query on the state of the collection
# we could have an SSE endpoint that implemented a hanging GET, allowing more data
# over the HTTP connection as it arrived
query = options.pop("query")
cursor = self.application.db[res_id].find(spec=query,
tailable=False,
await_data=False,
**options)
count = yield cursor.count(with_limit_and_skip=True)
if not count:
self.write('[]')
return
elif count >= 1:
self.write('[\n')
citem = 0
while (yield cursor.fetch_next):
resource = cursor.next_object()
self.write(
dumps_mongo(resource,
indent=2).replace('\\\\$',
'$').replace('$DOT$', '.'))
citem = citem + 1
if citem == count:
self.write('\n]')
else:
self.write(',\n')
yield self._add_response_headers(count)
self.set_status(200)
self.finish()
def _write_get(self, cursor, is_list=False, inline=False):
count = yield cursor.count(with_limit_and_skip=True)
is_list = count != 1 or is_list
if self.accept_content_type == MIME["PSBSON"]:
results = []
while (yield cursor.fetch_next):
resource = yield self._post_get(cursor.next_object(), inline)
results.append(resource)
if not is_list:
results = results[0]
else:
results = { "data": results }
self.write(bson_encode(results))
else:
if not count:
self.write('[]')
raise tornado.gen.Return(count)
elif is_list:
self.write('[\n')
# Write first entry
yield cursor.fetch_next
resource = cursor.next_object()
resource = yield self._post_get(resource, inline)
json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
while (yield cursor.fetch_next):
self.write(',\n')
resource = cursor.next_object()
resource = yield self._post_get(resource, inline)
json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
if is_list:
self.write('\n]')
raise tornado.gen.Return(count)
def get(self, res_id=None):
if not res_id:
self.write_error(403, message = "Data request must include a metadata id")
return
# Parse arguments and set up query
try:
parsed = yield self._parse_get_arguments()
options = dict(query = parsed["query"]["query"],
limit = parsed["limit"],
sort = parsed["sort"],
skip = parsed["skip"])
if not options["limit"]:
options.pop("limit", None)
if res_id:
options["query"][self.Id] = res_id
options["query"]["status"] = { "$ne": "DELETED" }
options["fields"] = { "_id": 0 }
except Exception as exp:
self.write_error(403, message = exp)
return
# we don't want to wait here since this is a direct query on the state of the collection
# we could have an SSE endpoint that implemented a hanging GET, allowing more data
# over the HTTP connection as it arrived
cursor = self.application.db[res_id].find(tailable=False, await_data=False, **options)
count = yield cursor.count(with_limit_and_skip=True)
if not count:
self.write('[]')
return
elif count >= 1:
self.write('[\n')
citem = 0
while (yield cursor.fetch_next):
resource = cursor.next_object()
self.write(dumps_mongo(resource, indent = 2).replace('\\\\$', '$').replace('$DOT$', '.'))
citem = citem + 1
if citem == count:
self.write('\n]')
else:
self.write(',\n')
yield self._add_response_headers(count)
self.set_status(201)
self.finish()
def _get_tailable(self, query, fields):
cursor = self._find(query = query,
fields = fields,
tailable = True,
await_data = True)
self.write('[')
first = True
while True:
if not cursor.alive:
self.write(']')
self.finish()
if (yield cursor.fetch_next):
if not first:
self.write(',')
else:
first = False
resource = cursor.next_object()
json_resonse = dumps_mongo(response, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
def _insert_extents(self, extent, parent):
http_client = AsyncHTTPClient()
extent["parent"] = {"href": parent,
"rel": "full"}
url = "{protocol}://{host}/extents"
return tornado.gen.Task(
http_client.fetch,
url.format(protocol = self.request.protocol,
host = self.request.host),
method = "POST",
body = dumps_mongo(extent),
request_timeout = 180,
validate_cert = False,
client_cert = settings.CLIENT_SSL_OPTIONS['certfile'],
client_key = settings.CLIENT_SSL_OPTIONS['keyfile'],
headers = { "Cache-Control": "no-cache",
"Content-Type": MIME["PSJSON"],
"connection": "close" }
)
def _post_return(self, resources):
# don't return data posts to measurement collectors
return
response = []
mids = {}
for resource in resources:
if resource["mid"] not in mids:
mids[resource["mid"]] = { "$or": [] }
mids[resource["mid"]]["$or"].append( { self.timestamp: resource[self.timestamp] } )
results = []
for mid, query in mids.iteritems():
results = yield self._return_resources(mid, query)
for result in results:
response.extend(result)
self.write(dumps_mongo(response, indent=2))
def _post_return(self, resources):
# don't return data posts to measurement collectors
return
response = []
mids = {}
for resource in resources:
if resource["mid"] not in mids:
mids[resource["mid"]] = {"$or": []}
mids[resource["mid"]]["$or"].append(
{self.timestamp: resource[self.timestamp]})
results = []
for mid, query in mids.iteritems():
results = yield self._return_resources(mid, query)
for result in results:
response.extend(result)
self.write(dumps_mongo(response, indent=2))
def _get_tailable(self, query, fields):
cursor = self._find(query=query,
fields=fields,
tailable=True,
await_data=True)
self.write('[')
first = True
while True:
if not cursor.alive:
self.write(']')
self.finish()
if (yield cursor.fetch_next):
if not first:
self.write(',')
else:
first = False
resource = cursor.next_object()
json_resonse = dumps_mongo(response, indent=2).replace(
'\\\\$', '$').replace('$DOT$', '.')
self.write(json_response)
def _write_get(self, cursor, is_list=False):
response = []
if cursor:
count = yield cursor.count()
if not count:
self.write("[]")
raise tornado.gen.Return(count)
while (yield cursor.fetch_next):
resource = cursor.next_object()
mid = resource["metadata_URL"].split("/")[resource["metadata_URL"].split("/").__len__() - 1]
try:
res = yield self.generate_response(mid)
except ValueError as exp:
raise ValueError(exp)
response.insert(0, res)
else:
count = 0
for d in self._query["$and"]:
if "mids" in d.keys():
if isinstance(d["mids"], dict):
for m in d["mids"]["$in"]:
try:
res = yield self.generate_response(m)
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
else:
try:
res = yield self.generate_response(d["mids"])
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
json_response = dumps_mongo(response, indent=2)
self.write(json_response)
raise tornado.gen.Return(count)
def _write_get(self, cursor, is_list=False, inline=False):
response = []
if cursor:
count = yield cursor.count()
while (yield cursor.fetch_next):
resource = cursor.next_object()
mid = resource["metadata_URL"].split('/')[
resource["metadata_URL"].split('/').__len__() - 1]
try:
res = yield self.generate_response(mid)
except ValueError as exp:
raise ValueError(exp)
response.insert(0, res)
else:
count = 0
for d in self._query["$and"]:
if 'mids' in d.keys():
if isinstance(d["mids"], dict):
for m in d['mids']['$in']:
try:
res = yield self.generate_response(m)
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
else:
try:
res = yield self.generate_response(d['mids'])
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
if self.accept_content_type == MIME["PSBSON"]:
json_response = bson_encode(response)
else:
json_response = dumps_mongo(response, indent=2)
self.write(json_response)
raise tornado.gen.Return(count)
def _write_get(self, cursor, is_list = False, inline=False):
response = []
if cursor:
count = yield cursor.count()
while (yield cursor.fetch_next):
resource = cursor.next_object()
mid = resource["metadata_URL"].split('/')[resource["metadata_URL"].split('/').__len__() - 1]
try:
res = yield self.generate_response(mid)
except ValueError as exp:
raise ValueError(exp)
response.insert(0, res)
else:
count = 0
for d in self._query["$and"]:
if 'mids' in d.keys():
if isinstance(d["mids"],dict):
for m in d['mids']['$in']:
try:
res = yield self.generate_response(m)
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
else:
try:
res = yield self.generate_response(d['mids'])
except ValueError as exp:
raise ValueError(exp)
count += 1
response.insert(0, res)
if self.accept_content_type == MIME["PSBSON"]:
json_response = bson_encode(response)
else:
json_response = dumps_mongo(response, indent=2)
self.write(json_response)
raise tornado.gen.Return(count)
