def _write_get(self, cursor, is_list = False): count = yield cursor.count(with_limit_and_skip=True) is_list = count > 1 or is_list if not count: self.write('[]') raise tornado.gen.Return(count) elif is_list: self.write('[\n') # Write first entry yield cursor.fetch_next resource = cursor.next_object() resource = yield self._post_get(resource) json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.') self.write(json_response) while (yield cursor.fetch_next): self.write(',\n') resource = cursor.next_object() resource = yield self._post_get(resource) json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.') self.write(json_response) if is_list: self.write('\n]') raise tornado.gen.Return(count)
def _create_child(self, key, resource): http_client = AsyncHTTPClient() url = "{protocol}://{host}{path}?validate=false&complete_links=false" links = [] query = [] for r in resource[key]: links.append(r) if "rel" in r and "href" in r else query.append(r) if query: response = yield tornado.gen.Task( http_client.fetch, url.format(protocol = self.request.protocol, host = self.request.host, path = self.reverse_url(key)), method = "POST", body = dumps_mongo(query), request_timeout = 180, validate_cert = False, client_cert = settings.CLIENT_SSL_OPTIONS['certfile'], client_key = settings.CLIENT_SSL_OPTIONS['keyfile'], headers = { "Cache-Control": "no-cache", "Content-Type": MIME["PSJSON"], "connection": "close" } ) if response.code >= 400: raise Exception("Could not add child resource") response = json.loads(response.body) if not isinstance(response, list): links.append({ "href": response["selfRef"], "rel": "full" }) else: for r in response: links.append({ "href": r["selfRef"], "rel": "full" }) raise tornado.gen.Return({ "collection": key, "hrefs": links })
def _create_child(self, key, resource): http_client = AsyncHTTPClient() url = "{protocol}://{host}{path}?validate=false&complete_links=false" links = [] query = [] for r in resource[key]: links.append(r) if "rel" in r and "href" in r else query.append(r) if query: response = yield tornado.gen.Task( http_client.fetch, url.format(protocol=self.request.protocol, host=self.request.host, path=self.reverse_url(key)), method="POST", body=dumps_mongo(query), request_timeout=180, validate_cert=False, client_cert=settings.CLIENT_SSL_OPTIONS['certfile'], client_key=settings.CLIENT_SSL_OPTIONS['keyfile'], headers={ "Cache-Control": "no-cache", "Content-Type": MIME["PSJSON"], "connection": "close" }) if response.code >= 400: raise Exception("Could not add child resource") response = json.loads(response.body) if not isinstance(response, list): links.append({"href": response["selfRef"], "rel": "full"}) else: for r in response: links.append({"href": r["selfRef"], "rel": "full"}) raise tornado.gen.Return({"collection": key, "hrefs": links})
def _return_resources(self, query): try: cursor = self._find(query = query) response = [] while (yield cursor.fetch_next): resource = ObjectDict._from_mongo(cursor.next_object()) response.append(resource) if len(response) == 1: location = self.request.full_url().split('?')[0] if not location.endswith(response[0][self.Id]): location = location + "/" + response[0][self.Id] self.set_header("Location", location) self.write(dumps_mongo(response[0], indent=2)) else: self.write(dumps_mongo(response, indent=2)) except Exception as exp: raise ValueError(exp)
def _return_resources(self, query): try: cursor = self._find(query=query) response = [] while (yield cursor.fetch_next): resource = ObjectDict._from_mongo(cursor.next_object()) response.append(resource) if len(response) == 1: location = self.request.full_url().split('?')[0] if not location.endswith(response[0][self.Id]): location = location + "/" + response[0][self.Id] self.set_header("Location", location) self.write(dumps_mongo(response[0], indent=2)) else: self.write(dumps_mongo(response, indent=2)) except Exception as exp: raise ValueError(exp)
def _write_get(self, cursor, is_list=False, inline=False): count = yield cursor.count(with_limit_and_skip=True) is_list = count != 1 or is_list if self.accept_content_type == MIME["PSBSON"]: results = [] while (yield cursor.fetch_next): resource = yield self._post_get(cursor.next_object(), inline) results.append(resource) if not is_list: results = results[0] else: results = {"data": results} self.write(bson_encode(results)) else: if not count: self.write('[]') raise tornado.gen.Return(count) elif is_list: self.write('[\n') # Write first entry yield cursor.fetch_next resource = cursor.next_object() resource = yield self._post_get(resource, inline) json_response = dumps_mongo(resource, indent=2).replace( '\\\\$', '$').replace('$DOT$', '.') self.write(json_response) while (yield cursor.fetch_next): self.write(',\n') resource = cursor.next_object() resource = yield self._post_get(resource, inline) json_response = dumps_mongo(resource, indent=2).replace( '\\\\$', '$').replace('$DOT$', '.') self.write(json_response) if is_list: self.write('\n]') raise tornado.gen.Return(count)
def get(self, res_id=None): if not res_id: self.write_error(403, message="Data request must include a metadata id") return # Parse arguments and set up query try: parsed = yield self._parse_get_arguments() options = dict(query=parsed["query"]["query"], limit=parsed["limit"], sort=parsed["sort"], skip=parsed["skip"]) if not options["limit"]: options.pop("limit", None) options["query"]["\\$status"] = {"$ne": "DELETED"} options["fields"] = {"_id": 0} except Exception as exp: self.write_error(403, message=exp) return # we don't want to wait here since this is a direct query on the state of the collection # we could have an SSE endpoint that implemented a hanging GET, allowing more data # over the HTTP connection as it arrived query = options.pop("query") cursor = self.application.db[res_id].find(spec=query, tailable=False, await_data=False, **options) count = yield cursor.count(with_limit_and_skip=True) if not count: self.write('[]') return elif count >= 1: self.write('[\n') citem = 0 while (yield cursor.fetch_next): resource = cursor.next_object() self.write( dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.')) citem = citem + 1 if citem == count: self.write('\n]') else: self.write(',\n') yield self._add_response_headers(count) self.set_status(200) self.finish()
def _write_get(self, cursor, is_list=False, inline=False): count = yield cursor.count(with_limit_and_skip=True) is_list = count != 1 or is_list if self.accept_content_type == MIME["PSBSON"]: results = [] while (yield cursor.fetch_next): resource = yield self._post_get(cursor.next_object(), inline) results.append(resource) if not is_list: results = results[0] else: results = { "data": results } self.write(bson_encode(results)) else: if not count: self.write('[]') raise tornado.gen.Return(count) elif is_list: self.write('[\n') # Write first entry yield cursor.fetch_next resource = cursor.next_object() resource = yield self._post_get(resource, inline) json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.') self.write(json_response) while (yield cursor.fetch_next): self.write(',\n') resource = cursor.next_object() resource = yield self._post_get(resource, inline) json_response = dumps_mongo(resource, indent=2).replace('\\\\$', '$').replace('$DOT$', '.') self.write(json_response) if is_list: self.write('\n]') raise tornado.gen.Return(count)
def get(self, res_id=None): if not res_id: self.write_error(403, message = "Data request must include a metadata id") return # Parse arguments and set up query try: parsed = yield self._parse_get_arguments() options = dict(query = parsed["query"]["query"], limit = parsed["limit"], sort = parsed["sort"], skip = parsed["skip"]) if not options["limit"]: options.pop("limit", None) if res_id: options["query"][self.Id] = res_id options["query"]["status"] = { "$ne": "DELETED" } options["fields"] = { "_id": 0 } except Exception as exp: self.write_error(403, message = exp) return # we don't want to wait here since this is a direct query on the state of the collection # we could have an SSE endpoint that implemented a hanging GET, allowing more data # over the HTTP connection as it arrived cursor = self.application.db[res_id].find(tailable=False, await_data=False, **options) count = yield cursor.count(with_limit_and_skip=True) if not count: self.write('[]') return elif count >= 1: self.write('[\n') citem = 0 while (yield cursor.fetch_next): resource = cursor.next_object() self.write(dumps_mongo(resource, indent = 2).replace('\\\\$', '$').replace('$DOT$', '.')) citem = citem + 1 if citem == count: self.write('\n]') else: self.write(',\n') yield self._add_response_headers(count) self.set_status(201) self.finish()
def _get_tailable(self, query, fields): cursor = self._find(query = query, fields = fields, tailable = True, await_data = True) self.write('[') first = True while True: if not cursor.alive: self.write(']') self.finish() if (yield cursor.fetch_next): if not first: self.write(',') else: first = False resource = cursor.next_object() json_resonse = dumps_mongo(response, indent=2).replace('\\\\$', '$').replace('$DOT$', '.') self.write(json_response)
def _insert_extents(self, extent, parent): http_client = AsyncHTTPClient() extent["parent"] = {"href": parent, "rel": "full"} url = "{protocol}://{host}/extents" return tornado.gen.Task( http_client.fetch, url.format(protocol = self.request.protocol, host = self.request.host), method = "POST", body = dumps_mongo(extent), request_timeout = 180, validate_cert = False, client_cert = settings.CLIENT_SSL_OPTIONS['certfile'], client_key = settings.CLIENT_SSL_OPTIONS['keyfile'], headers = { "Cache-Control": "no-cache", "Content-Type": MIME["PSJSON"], "connection": "close" } )
def _post_return(self, resources): # don't return data posts to measurement collectors return response = [] mids = {} for resource in resources: if resource["mid"] not in mids: mids[resource["mid"]] = { "$or": [] } mids[resource["mid"]]["$or"].append( { self.timestamp: resource[self.timestamp] } ) results = [] for mid, query in mids.iteritems(): results = yield self._return_resources(mid, query) for result in results: response.extend(result) self.write(dumps_mongo(response, indent=2))
def _post_return(self, resources): # don't return data posts to measurement collectors return response = [] mids = {} for resource in resources: if resource["mid"] not in mids: mids[resource["mid"]] = {"$or": []} mids[resource["mid"]]["$or"].append( {self.timestamp: resource[self.timestamp]}) results = [] for mid, query in mids.iteritems(): results = yield self._return_resources(mid, query) for result in results: response.extend(result) self.write(dumps_mongo(response, indent=2))
def _get_tailable(self, query, fields): cursor = self._find(query=query, fields=fields, tailable=True, await_data=True) self.write('[') first = True while True: if not cursor.alive: self.write(']') self.finish() if (yield cursor.fetch_next): if not first: self.write(',') else: first = False resource = cursor.next_object() json_resonse = dumps_mongo(response, indent=2).replace( '\\\\$', '$').replace('$DOT$', '.') self.write(json_response)
def _write_get(self, cursor, is_list=False): response = [] if cursor: count = yield cursor.count() if not count: self.write("[]") raise tornado.gen.Return(count) while (yield cursor.fetch_next): resource = cursor.next_object() mid = resource["metadata_URL"].split("/")[resource["metadata_URL"].split("/").__len__() - 1] try: res = yield self.generate_response(mid) except ValueError as exp: raise ValueError(exp) response.insert(0, res) else: count = 0 for d in self._query["$and"]: if "mids" in d.keys(): if isinstance(d["mids"], dict): for m in d["mids"]["$in"]: try: res = yield self.generate_response(m) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) else: try: res = yield self.generate_response(d["mids"]) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) json_response = dumps_mongo(response, indent=2) self.write(json_response) raise tornado.gen.Return(count)
def _write_get(self, cursor, is_list=False, inline=False): response = [] if cursor: count = yield cursor.count() while (yield cursor.fetch_next): resource = cursor.next_object() mid = resource["metadata_URL"].split('/')[ resource["metadata_URL"].split('/').__len__() - 1] try: res = yield self.generate_response(mid) except ValueError as exp: raise ValueError(exp) response.insert(0, res) else: count = 0 for d in self._query["$and"]: if 'mids' in d.keys(): if isinstance(d["mids"], dict): for m in d['mids']['$in']: try: res = yield self.generate_response(m) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) else: try: res = yield self.generate_response(d['mids']) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) if self.accept_content_type == MIME["PSBSON"]: json_response = bson_encode(response) else: json_response = dumps_mongo(response, indent=2) self.write(json_response) raise tornado.gen.Return(count)
def _write_get(self, cursor, is_list = False, inline=False): response = [] if cursor: count = yield cursor.count() while (yield cursor.fetch_next): resource = cursor.next_object() mid = resource["metadata_URL"].split('/')[resource["metadata_URL"].split('/').__len__() - 1] try: res = yield self.generate_response(mid) except ValueError as exp: raise ValueError(exp) response.insert(0, res) else: count = 0 for d in self._query["$and"]: if 'mids' in d.keys(): if isinstance(d["mids"],dict): for m in d['mids']['$in']: try: res = yield self.generate_response(m) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) else: try: res = yield self.generate_response(d['mids']) except ValueError as exp: raise ValueError(exp) count += 1 response.insert(0, res) if self.accept_content_type == MIME["PSBSON"]: json_response = bson_encode(response) else: json_response = dumps_mongo(response, indent=2) self.write(json_response) raise tornado.gen.Return(count)