def put(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path data = kwargs.get(b'data') if data == None: pass elif isinstance(data, Mapping): kwargs[b'data'] = data = convert.unicode2utf8(convert.value2json(data)) elif not isinstance(kwargs["data"], str): Log.error("data must be utf8 encoded string") if self.debug: sample = kwargs.get(b'data', "")[:300] Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample) # try: response = http.put(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:]) details = mo_json.json2value(utf82unicode(response.content)) if details.error: Log.error(convert.quote2string(details.error)) if details._shards.failed > 0: Log.error("Shard failures {{failures|indent}}", failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason) ) return details
def delete(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: response = http.delete(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130)) details = wrap(mo_json.json2value(utf82unicode(response.all_content))) if details.error: Log.error(details.error) return details except Exception, e: Log.error("Problem with call to {{url}}", url=url, cause=e)
def delete(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: response = http.delete(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130)) details = wrap(mo_json.json2value(utf82unicode(response.all_content))) if details.error: Log.error(details.error) return details except Exception as e: Log.error("Problem with call to {{url}}", url=url, cause=e)
def post(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate" data = kwargs.get(b'data') if data == None: pass elif isinstance(data, Mapping): kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data)) elif not isinstance(kwargs["data"], str): Log.error("data must be utf8 encoded string") if self.debug: sample = kwargs.get(b'data', "")[:300] Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample) if self.debug: Log.note("POST {{url}}", url=url) response = http.post(url, **kwargs) if response.status_code not in [200, 201]: Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000)) if self.debug: Log.note("response: {{response}}", response=utf82unicode(response.content)[:130]) details = mo_json.json2value(utf82unicode(response.content)) if details.error: Log.error(convert.quote2string(details.error)) if details._shards.failed > 0: Log.error("Shard failures {{failures|indent}}", failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason) ) return details except Exception as e: if url[0:4] != "http": suggestion = " (did you forget \"http://\" prefix on the host name?)" else: suggestion = "" if kwargs.get("data"): Log.error( "Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}", url=url, body=strings.limit(kwargs["data"], 100 if self.debug else 10000), cause=e ) else: Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def post(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path try: wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate" data = kwargs.get(b'data') if data == None: pass elif isinstance(data, Mapping): kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data)) elif not isinstance(kwargs["data"], str): Log.error("data must be utf8 encoded string") if self.debug: sample = kwargs.get(b'data', "")[:300] Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample) if self.debug: Log.note("POST {{url}}", url=url) response = http.post(url, **kwargs) if response.status_code not in [200, 201]: Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000)) if self.debug: Log.note("response: {{response}}", response=utf82unicode(response.content)[:130]) details = mo_json.json2value(utf82unicode(response.content)) if details.error: Log.error(convert.quote2string(details.error)) if details._shards.failed > 0: Log.error("Shard failures {{failures|indent}}", failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason) ) return details except Exception, e: if url[0:4] != "http": suggestion = " (did you forget \"http://\" prefix on the host name?)" else: suggestion = "" if kwargs.get("data"): Log.error( "Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}", url=url, body=strings.limit(kwargs["data"], 100 if self.debug else 10000), cause=e ) else: Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def delete_index(self, index_name): if not isinstance(index_name, unicode): Log.error("expecting an index name") if self.debug: Log.note("Deleting index {{index}}", index=index_name) # REMOVE ALL ALIASES TOO aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None] if aliases: self.post( path="/_aliases", data={"actions": [{"remove": a} for a in aliases]} ) url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name try: response = http.delete(url) if response.status_code != 200: Log.error("Expecting a 200, got {{code}}", code=response.status_code) details = mo_json.json2value(utf82unicode(response.content)) if self.debug: Log.note("delete response {{response}}", response=details) return response except Exception, e: Log.error("Problem with call to {{url}}", url=url, cause=e)
def delete_index(self, index_name): if not isinstance(index_name, unicode): Log.error("expecting an index name") if self.debug: Log.note("Deleting index {{index}}", index=index_name) # REMOVE ALL ALIASES TOO aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None] if aliases: self.post( path="/_aliases", data={"actions": [{"remove": a} for a in aliases]} ) url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name try: response = http.delete(url) if response.status_code != 200: Log.error("Expecting a 200, got {{code}}", code=response.status_code) details = mo_json.json2value(utf82unicode(response.content)) if self.debug: Log.note("delete response {{response}}", response=details) return response except Exception as e: Log.error("Problem with call to {{url}}", url=url, cause=e)
def _dict2json(value, sub_schema, path, net_new_properties, buffer): prefix = '{' for k, v in sort_using_key(value.items(), lambda r: r[0]): if v == None or v == '': continue append(buffer, prefix) prefix = COMMA if is_binary(k): k = utf82unicode(k) if not is_text(k): Log.error("Expecting property name to be a string") if k not in sub_schema: sub_schema[k] = {} net_new_properties.append(path + [k]) append(buffer, encode_basestring(encode_property(k))) append(buffer, COLON) typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer) if prefix is COMMA: append(buffer, COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}')
def test_multiple_agg_on_same_field(self): if self.not_real_service(): return test = wrap({ "query": { "from": { "type": "elasticsearch", "settings": { "host": ES_CLUSTER_LOCATION, "index": "unittest", "type": "test_result" } }, "select": [{ "name": "max_bytes", "value": "run.stats.bytes", "aggregate": "max" }, { "name": "count", "value": "run.stats.bytes", "aggregate": "count" }] } }) query = unicode2utf8(convert.value2json(test.query)) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def post_json(url, **kwargs): """ ASSUME RESPONSE IN IN JSON """ if 'json' in kwargs: kwargs['data'] = unicode2utf8(value2json(kwargs['json'])) del kwargs['json'] elif 'data' in kwargs: kwargs['data'] = unicode2utf8(value2json(kwargs['data'])) else: Log.error(u"Expecting `json` parameter") response = post(url, **kwargs) details = json2value(utf82unicode(response.content)) if response.status_code not in [200, 201, 202]: if "template" in details: Log.error(u"Bad response code {{code}}", code=response.status_code, cause=Except.wrap(details)) else: Log.error(u"Bad response code {{code}}\n{{details}}", code=response.status_code, details=details) else: return details
def test_timing(self): if self.not_real_service(): return test = wrap({ "query": { "from": { "type": "elasticsearch", "settings": { "host": ES_CLUSTER_LOCATION, "index": "unittest", "type": "test_result" } }, "select": [{ "name": "count", "value": "run.duration", "aggregate": "count" }, { "name": "total", "value": "run.duration", "aggregate": "sum" }], "edges": [{ "name": "chunk", "value": ["run.suite", "run.chunk"] }, "result.ok"], "where": { "and": [{ "lt": { "timestamp": Date.floor(Date.now()).milli / 1000 } }, { "gte": { "timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000 } }] }, "format": "cube", "samples": { "limit": 30 } } }) query = unicode2utf8(convert.value2json(test.query)) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def error(response): response = utf82unicode(response.content) try: e = Except.new_instance(json2value(response)) except Exception: e = None if e: Log.error("Failed request", e) else: Log.error("Failed request\n {{response}}", {"response": response})
def test_simple_query(self): if self.not_real_service(): return query = unicode2utf8(convert.value2json({"from": "unittest"})) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def get_json(url, **kwargs): """ ASSUME RESPONSE IN IN JSON """ response = get(url, **kwargs) try: c = response.all_content return json2value(utf82unicode(c)) except Exception as e: if mo_math.round(response.status_code, decimal=-2) in [400, 500]: Log.error(u"Bad GET response: {{code}}", code=response.status_code) else: Log.error(u"Good GET requests, but bad JSON", cause=e)
def get_json(url, **kwargs): """ ASSUME RESPONSE IN IN JSON """ response = get(url, **kwargs) try: c = response.all_content return json2value(utf82unicode(c)) except Exception as e: if Math.round(response.status_code, decimal=-2) in [400, 500]: Log.error(u"Bad GET response: {{code}}", code=response.status_code) else: Log.error(u"Good GET requests, but bad JSON", cause=e)
def _dict2json(value, _buffer): prefix = u'{"$object": ".", "' for k, v in value.iteritems(): append(_buffer, prefix) prefix = u", \"" if isinstance(k, str): k = utf82unicode(k) if not isinstance(k, unicode): Log.error("Expecting property name to be a string") for c in k: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u"\": ") _typed_encode(v, _buffer) append(_buffer, u"}")
def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key=key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return utf82unicode(source.read()).split("\n") if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: return LazyLines(source)
def set_refresh_interval(self, seconds, **kwargs): """ :param seconds: -1 FOR NO REFRESH :param kwargs: ANY OTHER REQUEST PARAMETERS :return: None """ if seconds <= 0: interval = -1 else: interval = unicode(seconds) + "s" if self.cluster.version.startswith("0.90."): response = self.cluster.put( "/" + self.settings.index + "/_settings", data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}', **kwargs ) result = mo_json.json2value(utf82unicode(response.all_content)) if not result.ok: Log.error("Can not set refresh interval ({{error}})", { "error": utf82unicode(response.all_content) }) elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])): response = self.cluster.put( "/" + self.settings.index + "/_settings", data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'), **kwargs ) result = mo_json.json2value(utf82unicode(response.all_content)) if not result.acknowledged: Log.error("Can not set refresh interval ({{error}})", { "error": utf82unicode(response.all_content) }) else: Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
def put(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path if self.debug: sample = kwargs["data"][:300] Log.note("PUT {{url}}:\n{{data|indent}}", url=url, data=sample) try: response = http.put(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:]) return response except Exception, e: Log.error("Problem with call to {{url}}", url= url, cause=e)
def put(self, path, **kwargs): url = self.settings.host + ":" + unicode(self.settings.port) + path if self.debug: sample = kwargs["data"][:300] Log.note("PUT {{url}}:\n{{data|indent}}", url=url, data=sample) try: response = http.put(url, **kwargs) if response.status_code not in [200]: Log.error(response.reason+": "+response.all_content) if self.debug: Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:]) return response except Exception as e: Log.error("Problem with call to {{url}}", url= url, cause=e)
def get(*args, **kwargs): body = kwargs.get("data") if not body: return wrap({"status_code": 400}) text = utf82unicode(body) data = json2value(text) result = jx.run(data) output_bytes = unicode2utf8(value2json(result)) return wrap({ "status_code": 200, "all_content": output_bytes, "content": output_bytes })
def execute_query(self, query): query = wrap(query) try: query = unicode2utf8(value2json(query)) # EXECUTE QUERY response = self.try_till_response(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) return result except Exception as e: Log.error("Failed query", e)
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception as e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return utf82unicode(json)
def annotation(): if flask.request.headers.get("content-length", "") in ["", "0"]: # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK return Response(ERROR_CONTENT, status=400, headers={"Content-Type": "text/html"}) elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT: Log.error("Query is too large to parse") request_body = flask.request.get_data().strip() text = utf82unicode(request_body) data = json2value(text) try: record_request(flask.request, data, None, None) except Exception as e: Log.error("Problem processing request {{request}}")
def _dict2json(value, _buffer): try: prefix = u"{\"" for k, v in value.iteritems(): append(_buffer, prefix) prefix = u", \"" if isinstance(k, str): k = utf82unicode(k) for c in k: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u"\": ") _value2json(v, _buffer) append(_buffer, u"}") except Exception as e: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer): try: prefix = u"{\"" for k, v in value.items(): append(_buffer, prefix) prefix = COMMA_QUOTE if is_binary(k): k = utf82unicode(k) for c in k: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, QUOTE_COLON) _value2json(v, _buffer) append(_buffer, u"}") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def send_queries(self, subtest, places=6): subtest = wrap(subtest) try: # EXECUTE QUERY num_expectations = 0 for i, (k, v) in enumerate(subtest.items()): if k.startswith("expecting_"): # WHAT FORMAT ARE WE REQUESTING format = k[len("expecting_"):] elif k == "expecting": # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS) format = None else: continue num_expectations += 1 expected = v subtest.query.format = format subtest.query.meta.testing = ( num_expectations == 1 ) # MARK FIRST QUERY FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION query = unicode2utf8(value2json(subtest.query)) # EXECUTE QUERY response = self.try_till_response(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) container = jx_elasticsearch.new_instance( self._es_test_settings) query = QueryOp.wrap(subtest.query, container, container.namespace) compare_to_expected(query, result, expected, places) Log.note("PASS {{name|quote}} (format={{format}})", name=subtest.name, format=format) if num_expectations == 0: Log.error( "Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses", name=subtest.name) except Exception as e: Log.error("Failed test {{name|quote}}", {"name": subtest.name}, e)
def test_longest_running_tests(self): test = wrap({ "query": { "sort": { "sort": -1, "field": "avg" }, "from": { "from": "unittest", "where": { "and": [{ "gt": { "build.date": "1439337600" } }] }, "groupby": [ "build.platform", "build.type", "run.suite", "result.test" ], "select": [{ "aggregate": "avg", "name": "avg", "value": "result.duration" }], "format": "table", "limit": 100 }, "limit": 100, "format": "list" } }) query = unicode2utf8(convert.value2json(test.query)) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def test_branch_count(self): if self.not_real_service(): return test = wrap({ "query": { "from": { "type": "elasticsearch", "settings": { "host": ES_CLUSTER_LOCATION, "index": "unittest", "type": "test_result" } }, "select": [ { "aggregate": "count" }, ], "edges": ["build.branch"], "where": { "or": [{ "missing": "build.id" } # {"gte": {"timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000}} ] }, "format": "table" } }) query = unicode2utf8(convert.value2json(test.query)) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def _dict2json(self, value, sub_schema, path, net_new_properties, _buffer): prefix = '{' for k, v in ((kk, value[kk]) for kk in sorted(value.keys())): if v == None or v == '': continue append(_buffer, prefix) prefix = COMMA if isinstance(k, binary_type): k = utf82unicode(k) if not isinstance(k, text_type): Log.error("Expecting property name to be a string") if k not in sub_schema: sub_schema[k] = {} net_new_properties.append(path+[k]) append(_buffer, json.dumps(encode_property(k))) append(_buffer, COLON) self._typed_encode(v, sub_schema[k], path+[k], net_new_properties, _buffer) if prefix == COMMA: append(_buffer, COMMA+QUOTED_EXISTS_TYPE+COLON+'1}') else: append(_buffer, '{'+QUOTED_EXISTS_TYPE+COLON+'0}')
def post_json(url, **kwargs): """ ASSUME RESPONSE IN IN JSON """ if 'json' in kwargs: kwargs['data'] = unicode2utf8(value2json(kwargs['json'])) elif 'data' in kwargs: kwargs['data'] = unicode2utf8(value2json(kwargs['data'])) else: Log.error(u"Expecting `json` parameter") response = post(url, **kwargs) c = response.content try: details = json2value(utf82unicode(c)) except Exception as e: Log.error(u"Unexpected return value {{content}}", content=c, cause=e) if response.status_code not in [200, 201]: Log.error(u"Bad response", cause=Except.wrap(details)) return details
def test_failures_by_directory(self): if self.not_real_service(): return test = wrap({ "query": { "from": { "type": "elasticsearch", "settings": { "host": ES_CLUSTER_LOCATION, "index": "unittest", "type": "test_result" } }, "select": [{ "aggregate": "count" }], "edges": ["result.test", "result.ok"], "where": { "prefix": { "result.test": "/" } }, "format": "table" } }) query = unicode2utf8(convert.value2json(test.query)) # EXECUTE QUERY with Timer("query"): response = http.get(self.testing.query, data=query) if response.status_code != 200: error(response) result = json2value(utf82unicode(response.all_content)) Log.note("result\n{{result|indent}}", {"result": result})
def pretty_json(value): try: if value is False: return "false" elif value is True: return "true" elif isinstance(value, Mapping): try: items = sort_using_key(list(value.items()), lambda r: r[0]) values = [ encode_basestring(k) + PRETTY_COLON + indent(pretty_json(v)).strip() for k, v in items if v != None ] if not values: return "{}" elif len(values) == 1: return "{" + values[0] + "}" else: return "{\n" + INDENT + (",\n" + INDENT).join(values) + "\n}" except Exception as e: from mo_logs import Log from mo_math import OR if OR(not isinstance(k, text_type) for k in value.keys()): Log.error("JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e) Log.error("problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e) elif value in (None, Null): return "null" elif isinstance(value, (text_type, binary_type)): if isinstance(value, binary_type): value = utf82unicode(value) try: return quote(value) except Exception as e: from mo_logs import Log try: Log.note( "try explicit convert of string with length {{length}}", length=len(value)) acc = [QUOTE] for c in value: try: try: c2 = ESCAPE_DCT[c] except Exception: c2 = c c3 = text_type(c2) acc.append(c3) except BaseException: pass # Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g) acc.append(QUOTE) output = u"".join(acc) Log.note("return value of length {{length}}", length=len(output)) return output except BaseException as f: Log.warning("can not even explicit convert {{type}}", type=f.__class__.__name__, cause=f) return "null" elif isinstance(value, list): if not value: return "[]" if ARRAY_MAX_COLUMNS == 1: return "[\n" + ",\n".join( [indent(pretty_json(v)) for v in value]) + "\n]" if len(value) == 1: j = pretty_json(value[0]) if j.find("\n") >= 0: return "[\n" + indent(j) + "\n]" else: return "[" + j + "]" js = [pretty_json(v) for v in value] max_len = max(*[len(j) for j in js]) if max_len <= ARRAY_ITEM_MAX_LENGTH and max( *[j.find("\n") for j in js]) == -1: # ALL TINY VALUES num_columns = max( 1, min( ARRAY_MAX_COLUMNS, int( floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW return "[" + PRETTY_COMMA.join(js) + "]" if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN return "[\n" + ",\n".join( [indent(pretty_json(v)) for v in value]) + "\n]" content = ",\n".join( PRETTY_COMMA.join( j.rjust(max_len) for j in js[r:r + num_columns]) for r in xrange(0, len(js), num_columns)) return "[\n" + indent(content) + "\n]" pretty_list = js output = ["[\n"] for i, p in enumerate(pretty_list): try: if i > 0: output.append(",\n") output.append(indent(p)) except Exception: from mo_logs import Log Log.warning( "problem concatenating string of length {{len1}} and {{len2}}", len1=len("".join(output)), len2=len(p)) output.append("\n]") try: return "".join(output) except Exception as e: from mo_logs import Log Log.error("not expected", cause=e) elif hasattr(value, '__data__'): d = value.__data__() return pretty_json(d) elif hasattr(value, '__json__'): j = value.__json__() if j == None: return " null " # TODO: FIND OUT WHAT CAUSES THIS return pretty_json(json_decoder(j)) elif scrub(value) is None: return "null" elif hasattr(value, '__iter__'): return pretty_json(list(value)) elif hasattr(value, '__call__'): return "null" else: try: if int(value) == value: return text_type(int(value)) except Exception: pass try: if float(value) == value: return text_type(float(value)) except Exception: pass return pypy_json_encode(value) except Exception as e: problem_serializing(value, e)
def _value2json(value, _buffer): try: _class = value.__class__ if value is None: append(_buffer, u"null") return elif value is True: append(_buffer, u"true") return elif value is False: append(_buffer, u"false") return type = value.__class__ if type is binary_type: append(_buffer, QUOTE) try: v = utf82unicode(value) except Exception as e: problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, QUOTE) elif type is text_type: append(_buffer, QUOTE) for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, QUOTE) elif type is dict: if not value: append(_buffer, u"{}") else: _dict2json(value, _buffer) return elif type is Data: d = _get(value, "_dict") # MIGHT BE A VALUE NOT A DICT _value2json(d, _buffer) return elif type in (int, long, Decimal): append(_buffer, text_type(value)) elif type is float: if math.isnan(value) or math.isinf(value): append(_buffer, u'null') else: append(_buffer, float2json(value)) elif type in (set, list, tuple, FlatList): _list2json(value, _buffer) elif type is date: append(_buffer, float2json(time.mktime(value.timetuple()))) elif type is datetime: append(_buffer, float2json(time.mktime(value.timetuple()))) elif type is Date: append(_buffer, float2json(value.unix)) elif type is timedelta: append(_buffer, float2json(value.total_seconds())) elif type is Duration: append(_buffer, float2json(value.seconds)) elif type is NullType: append(_buffer, u"null") elif isinstance(value, Mapping): if not value: append(_buffer, u"{}") else: _dict2json(value, _buffer) return elif hasattr(value, '__data__'): d = value.__data__() _value2json(d, _buffer) elif hasattr(value, '__json__'): j = value.__json__() append(_buffer, j) elif hasattr(value, '__iter__'): _iter2json(value, _buffer) else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def pretty_json(value): try: if value is False: return "false" elif value is True: return "true" elif is_data(value): try: items = sort_using_key(value.items(), lambda r: r[0]) values = [encode_basestring(k) + PRETTY_COLON + pretty_json(v) for k, v in items if v != None] if not values: return "{}" elif len(values) == 1: return "{" + values[0] + "}" else: return "{\n" + ",\n".join(indent(v) for v in values) + "\n}" except Exception as e: from mo_logs import Log from mo_math import OR if OR(not is_text(k) for k in value.keys()): Log.error( "JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e ) Log.error( "problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e ) elif value in (None, Null): return "null" elif value.__class__ in (binary_type, text_type): if is_binary(value): value = utf82unicode(value) try: return quote(value) except Exception as e: from mo_logs import Log try: Log.note("try explicit convert of string with length {{length}}", length=len(value)) acc = [QUOTE] for c in value: try: try: c2 = ESCAPE_DCT[c] except Exception: c2 = c c3 = text_type(c2) acc.append(c3) except BaseException: pass # Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g) acc.append(QUOTE) output = u"".join(acc) Log.note("return value of length {{length}}", length=len(output)) return output except BaseException as f: Log.warning("can not convert {{type}} to json", type=f.__class__.__name__, cause=f) return "null" elif is_list(value): if not value: return "[]" if ARRAY_MAX_COLUMNS == 1: return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" if len(value) == 1: j = pretty_json(value[0]) if j.find("\n") >= 0: return "[\n" + indent(j) + "\n]" else: return "[" + j + "]" js = [pretty_json(v) for v in value] max_len = max(*[len(j) for j in js]) if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1: # ALL TINY VALUES num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW return "[" + PRETTY_COMMA.join(js) + "]" if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" content = ",\n".join( PRETTY_COMMA.join(j.rjust(max_len) for j in js[r:r + num_columns]) for r in xrange(0, len(js), num_columns) ) return "[\n" + indent(content) + "\n]" pretty_list = js output = ["[\n"] for i, p in enumerate(pretty_list): try: if i > 0: output.append(",\n") output.append(indent(p)) except Exception: from mo_logs import Log Log.warning("problem concatenating string of length {{len1}} and {{len2}}", len1=len("".join(output)), len2=len(p) ) output.append("\n]") try: return "".join(output) except Exception as e: from mo_logs import Log Log.error("not expected", cause=e) elif hasattr(value, '__data__'): d = value.__data__() return pretty_json(d) elif hasattr(value, '__json__'): j = value.__json__() if j == None: return " null " # TODO: FIND OUT WHAT CAUSES THIS return pretty_json(json_decoder(j)) elif scrub(value) is None: return "null" elif hasattr(value, '__iter__'): return pretty_json(list(value)) elif hasattr(value, '__call__'): return "null" else: try: if int(value) == value: return text_type(int(value)) except Exception: pass try: if float(value) == value: return text_type(float(value)) except Exception: pass return pypy_json_encode(value) except Exception as e: problem_serializing(value, e)
def pretty_json(value): try: if value is False: return "false" elif value is True: return "true" elif isinstance(value, Mapping): try: if not value: return "{}" items = list(value.items()) if len(items) == 1: return "{" + unicode_key(items[0][0]) + ": " + pretty_json(items[0][1]).strip() + "}" items = sorted(items, lambda a, b: value_compare(a[0], b[0])) values = [unicode_key(k) + ": " + indent(pretty_json(v)).strip() for k, v in items if v != None] return "{\n" + INDENT + (",\n" + INDENT).join(values) + "\n}" except Exception as e: from mo_logs import Log from mo_math import OR if OR(not isinstance(k, basestring) for k in value.keys()): Log.error( "JSON must have string keys: {{keys}}:", keys=[k for k in value.keys()], cause=e ) Log.error( "problem making dict pretty: keys={{keys}}:", keys=[k for k in value.keys()], cause=e ) elif value in (None, Null): return "null" elif isinstance(value, basestring): if isinstance(value, str): value = utf82unicode(value) try: return quote(value) except Exception as e: from mo_logs import Log try: Log.note("try explicit convert of string with length {{length}}", length=len(value)) acc = [u"\""] for c in value: try: try: c2 = ESCAPE_DCT[c] except Exception: c2 = c c3 = unicode(c2) acc.append(c3) except BaseException: pass # Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g) acc.append(u"\"") output = u"".join(acc) Log.note("return value of length {{length}}", length=len(output)) return output except BaseException, f: Log.warning("can not even explicit convert {{type}}", type=f.__class__.__name__, cause=f) return "null" elif isinstance(value, list): if not value: return "[]" if ARRAY_MAX_COLUMNS == 1: return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" if len(value) == 1: j = pretty_json(value[0]) if j.find("\n") >= 0: return "[\n" + indent(j) + "\n]" else: return "[" + j + "]" js = [pretty_json(v) for v in value] max_len = max(*[len(j) for j in js]) if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1: # ALL TINY VALUES num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW return "[" + ", ".join(js) + "]" if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]" content = ",\n".join( ", ".join(j.rjust(max_len) for j in js[r:r + num_columns]) for r in xrange(0, len(js), num_columns) ) return "[\n" + indent(content) + "\n]" pretty_list = js output = ["[\n"] for i, p in enumerate(pretty_list): try: if i > 0: output.append(",\n") output.append(indent(p)) except Exception: from mo_logs import Log Log.warning("problem concatenating string of length {{len1}} and {{len2}}", len1=len("".join(output)), len2=len(p) ) output.append("\n]") try: return "".join(output) except Exception as e: from mo_logs import Log Log.error("not expected", cause=e)
def _value2json(value, _buffer): try: _class = value.__class__ if value is None: append(_buffer, u"null") return elif value is True: append(_buffer, u"true") return elif value is False: append(_buffer, u"false") return type = value.__class__ if type is binary_type: append(_buffer, QUOTE) try: v = utf82unicode(value) except Exception as e: problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, QUOTE) elif type is text_type: append(_buffer, QUOTE) for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, QUOTE) elif type is dict: if not value: append(_buffer, u"{}") else: _dict2json(value, _buffer) return elif type is Data: d = _get(value, SLOT) # MIGHT BE A VALUE NOT A DICT _value2json(d, _buffer) return elif type in (int, long, Decimal): append(_buffer, text_type(value)) elif type is float: if math.isnan(value) or math.isinf(value): append(_buffer, u'null') else: append(_buffer, float2json(value)) elif type in (set, list, tuple, FlatList): _list2json(value, _buffer) elif type is date: append(_buffer, float2json(time.mktime(value.timetuple()))) elif type is datetime: append(_buffer, float2json(time.mktime(value.timetuple()))) elif type is Date: append(_buffer, float2json(value.unix)) elif type is timedelta: append(_buffer, float2json(value.total_seconds())) elif type is Duration: append(_buffer, float2json(value.seconds)) elif type is NullType: append(_buffer, u"null") elif is_data(value): if not value: append(_buffer, u"{}") else: _dict2json(value, _buffer) return elif hasattr(value, '__data__'): d = value.__data__() _value2json(d, _buffer) elif hasattr(value, '__json__'): j = value.__json__() append(_buffer, j) elif hasattr(value, '__iter__'): _iter2json(value, _buffer) else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(value, _buffer): try: if value is None: append(_buffer, u'{"$value": null}') return elif value is True: append(_buffer, u'{"$value": true}') return elif value is False: append(_buffer, u'{"$value": false}') return _type = value.__class__ if _type in (dict, Data): if value: _dict2json(value, _buffer) else: append(_buffer, u'{"$object": "."}') elif _type is str: append(_buffer, u'{"$value": "') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type is unicode: append(_buffer, u'{"$value": "') for c in value: append(_buffer, ESCAPE_DCT.get(c, c)) append(_buffer, u'"}') elif _type in (int, long, Decimal): append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type is float: append(_buffer, u'{"$value": ') append(_buffer, float2json(value)) append(_buffer, u'}') elif _type in (set, list, tuple, FlatList): _list2json(value, _buffer) elif _type is date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is datetime: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.timetuple()))) append(_buffer, u'}') elif _type is Date: append(_buffer, u'{"$value": ') append(_buffer, float2json(time.mktime(value.value.timetuple()))) append(_buffer, u'}') elif _type is timedelta: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.total_seconds())) append(_buffer, u'}') elif _type is Duration: append(_buffer, u'{"$value": ') append(_buffer, float2json(value.seconds)) append(_buffer, u'}') elif _type is NullType: append(_buffer, u"null") elif hasattr(value, '__json__'): j = value.__json__() t = json2typed(j) append(_buffer, t) elif hasattr(value, '__iter__'): _iter2json(value, _buffer) else: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(_repr(value) + " is not JSON serializable", e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer): """ :param value: THE DATA STRUCTURE TO ENCODE :param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE :param path: list OF CURRENT PATH :param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema :param buffer: UnicodeBuilder OBJECT :return: """ try: # from jx_base import Column if sub_schema.__class__.__name__=='Column': value_json_type = python_type_to_json_type[value.__class__] column_json_type = es_type_to_json_type[sub_schema.es_type] if value_json_type == column_json_type: pass # ok elif value_json_type == NESTED and all(python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None): pass # empty arrays can be anything else: from mo_logs import Log Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.name) sub_schema = {json_type_to_inserter_type[value_json_type]: sub_schema} if value == None: from mo_logs import Log Log.error("can not encode null (missing) values") elif value is True: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'true}') return elif value is False: if BOOLEAN_TYPE not in sub_schema: sub_schema[BOOLEAN_TYPE] = {} net_new_properties.append(path + [BOOLEAN_TYPE]) append(buffer, '{') append(buffer, QUOTED_BOOLEAN_TYPE) append(buffer, 'false}') return _type = value.__class__ if _type in (dict, Data): if sub_schema.__class__.__name__ == 'Column': from mo_logs import Log Log.error("Can not handle {{column|json}}", column=sub_schema) if NESTED_TYPE in sub_schema: # PREFER NESTED, WHEN SEEN BEFORE if value: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[') _dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, ']' + COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, text_type(len(value))) append(buffer, '}') else: # SINGLETON LIST append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}]') append(buffer, COMMA) append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') else: if EXISTS_TYPE not in sub_schema: sub_schema[EXISTS_TYPE] = {} net_new_properties.append(path + [EXISTS_TYPE]) if value: _dict2json(value, sub_schema, path, net_new_properties, buffer) else: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '1}') elif _type is binary_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') try: v = utf82unicode(value) except Exception as e: raise problem_serializing(value, e) for c in v: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type is text_type: if STRING_TYPE not in sub_schema: sub_schema[STRING_TYPE] = True net_new_properties.append(path + [STRING_TYPE]) append(buffer, '{') append(buffer, QUOTED_STRING_TYPE) append(buffer, '"') for c in value: append(buffer, ESCAPE_DCT.get(c, c)) append(buffer, '"}') elif _type in (int, long): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, text_type(value)) append(buffer, '}') elif _type in (float, Decimal): if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value)) append(buffer, '}') elif _type in (set, list, tuple, FlatList): if len(value) == 0: append(buffer, '{') append(buffer, QUOTED_EXISTS_TYPE) append(buffer, '0}') elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList) for v in value): # THIS IS NOT DONE BECAUSE if len(value) == 1: if NESTED_TYPE in sub_schema: append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: # NO NEED TO NEST, SO DO NOT DO IT typed_encode(value[0], sub_schema, path, net_new_properties, buffer) else: if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: # ALLOW PRIMITIVE MULTIVALUES value = [v for v in value if v != None] types = list(set(json_type_to_inserter_type[python_type_to_json_type[v.__class__]] for v in value)) if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) append(buffer, '[]}') elif len(types) > 1: _list2json(value, sub_schema, path + [NESTED_TYPE], net_new_properties, buffer) else: element_type = types[0] if element_type not in sub_schema: sub_schema[element_type] = True net_new_properties.append(path + [element_type]) append(buffer, '{') append(buffer, quote(element_type)) append(buffer, COLON) _multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, buffer) append(buffer, '}') elif _type is date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is datetime: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(time.mktime(value.timetuple()))) append(buffer, '}') elif _type is Date: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.unix)) append(buffer, '}') elif _type is timedelta: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.total_seconds())) append(buffer, '}') elif _type is Duration: if NUMBER_TYPE not in sub_schema: sub_schema[NUMBER_TYPE] = True net_new_properties.append(path + [NUMBER_TYPE]) append(buffer, '{') append(buffer, QUOTED_NUMBER_TYPE) append(buffer, float2json(value.seconds)) append(buffer, '}') elif _type is NullType: append(buffer, 'null') elif hasattr(value, '__data__'): typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer) elif hasattr(value, '__iter__'): if NESTED_TYPE not in sub_schema: sub_schema[NESTED_TYPE] = {} net_new_properties.append(path + [NESTED_TYPE]) append(buffer, '{') append(buffer, QUOTED_NESTED_TYPE) _iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer) append(buffer, '}') else: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable") except Exception as e: from mo_logs import Log Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def tuid_endpoint(path): try: if flask.request.headers.get("content-length", "") in ["", "0"]: # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK return Response(EXPECTING_QUERY, status=400, headers={"Content-Type": "text/html"}) elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT: return Response(unicode2utf8("request too large"), status=400, headers={"Content-Type": "text/html"}) request_body = flask.request.get_data().strip() query = json2value(utf82unicode(request_body)) # ENSURE THE QUERY HAS THE CORRECT FORM if query['from'] != 'files': Log.error("Can only handle queries on the `files` table") ands = listwrap(query.where['and']) if len(ands) != 3: Log.error( 'expecting a simple where clause with following structure\n{{example|json}}', example={ "and": [{ "eq": { "branch": "<BRANCH>" } }, { "eq": { "revision": "<REVISION>" } }, { "in": { "path": ["<path1>", "<path2>", "...", "<pathN>"] } }] }) rev = None paths = None branch_name = None for a in ands: rev = coalesce(rev, a.eq.revision) paths = unwraplist(coalesce(paths, a['in'].path, a.eq.path)) branch_name = coalesce(branch_name, a.eq.branch) paths = listwrap(paths) if len(paths) == 0: response, completed = [], True elif service.conn.pending_transactions > TOO_BUSY: # CHECK IF service IS VERY BUSY # TODO: BE SURE TO UPDATE STATS TOO Log.note("Too many open transactions") response, completed = [], False else: # RETURN TUIDS with Timer("tuid internal response time for {{num}} files", {"num": len(paths)}): response, completed = service.get_tuids_from_files( revision=rev, files=paths, going_forward=True, repo=branch_name) if not completed: Log.note( "Request for {{num}} files is incomplete for revision {{rev}}.", num=len(paths), rev=rev) if query.meta.format == 'list': formatter = _stream_list else: formatter = _stream_table return Response(formatter(response), status=200 if completed else 202, headers={"Content-Type": mimetype.JSON}) except Exception as e: e = Except.wrap(e) Log.warning("could not handle request", cause=e) return Response(unicode2utf8(value2json(e, pretty=True)), status=400, headers={"Content-Type": "text/html"})
def tuid_endpoint(path): with RegisterThread(): try: service.statsdaemon.update_requests(requests_total=1) if flask.request.headers.get("content-length", "") in ["", "0"]: # ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK service.statsdaemon.update_requests(requests_complete=1, requests_passed=1) return Response( EXPECTING_QUERY, status=400, headers={ "Content-Type": "text/html" } ) elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT: service.statsdaemon.update_requests(requests_complete=1, requests_passed=1) return Response( unicode2utf8("request too large"), status=400, headers={ "Content-Type": "text/html" } ) request_body = flask.request.get_data().strip() query = json2value(utf82unicode(request_body)) # ENSURE THE QUERY HAS THE CORRECT FORM if query['from'] != 'files': Log.error("Can only handle queries on the `files` table") ands = listwrap(query.where['and']) if len(ands) != 3: Log.error( 'expecting a simple where clause with following structure\n{{example|json}}', example={"and": [ {"eq": {"branch": "<BRANCH>"}}, {"eq": {"revision": "<REVISION>"}}, {"in": {"path": ["<path1>", "<path2>", "...", "<pathN>"]}} ]} ) rev = None paths = None branch_name = None for a in ands: rev = coalesce(rev, a.eq.revision) paths = unwraplist(coalesce(paths, a['in'].path, a.eq.path)) branch_name = coalesce(branch_name, a.eq.branch) paths = listwrap(paths) if len(paths) == 0: response, completed = [], True elif service.conn.pending_transactions > TOO_BUSY: # CHECK IF service IS VERY BUSY # TODO: BE SURE TO UPDATE STATS TOO Log.note("Too many open transactions") response, completed = [], False elif service.get_thread_count() > TOO_MANY_THREADS: Log.note("Too many threads open") response, completed = [], False else: # RETURN TUIDS with Timer("tuid internal response time for {{num}} files", {"num": len(paths)}): response, completed = service.get_tuids_from_files( revision=rev, files=paths, going_forward=True, repo=branch_name ) if not completed: Log.note( "Request for {{num}} files is incomplete for revision {{rev}}.", num=len(paths), rev=rev ) if query.meta.format == 'list': formatter = _stream_list else: formatter = _stream_table service.statsdaemon.update_requests( requests_complete=1 if completed else 0, requests_incomplete=1 if not completed else 0, requests_passed=1 ) return Response( formatter(response), status=200 if completed else 202, headers={ "Content-Type": "application/json" } ) except Exception as e: e = Except.wrap(e) service.statsdaemon.update_requests(requests_incomplete=1, requests_failed=1) Log.warning("could not handle request", cause=e) return Response( unicode2utf8(value2json(e, pretty=True)), status=400, headers={ "Content-Type": "text/html" } )