def put(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data = convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
# try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:])
details = mo_json.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
def delete(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.delete(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(mo_json.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def delete(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
response = http.delete(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
details = wrap(mo_json.json2value(utf82unicode(response.all_content)))
if details.error:
Log.error(details.error)
return details
except Exception as e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = mo_json.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception as e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def post(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
try:
wrap(kwargs).headers["Accept-Encoding"] = "gzip,deflate"
data = kwargs.get(b'data')
if data == None:
pass
elif isinstance(data, Mapping):
kwargs[b'data'] = data =convert.unicode2utf8(convert.value2json(data))
elif not isinstance(kwargs["data"], str):
Log.error("data must be utf8 encoded string")
if self.debug:
sample = kwargs.get(b'data', "")[:300]
Log.note("{{url}}:\n{{data|indent}}", url=url, data=sample)
if self.debug:
Log.note("POST {{url}}", url=url)
response = http.post(url, **kwargs)
if response.status_code not in [200, 201]:
Log.error(response.reason.decode("latin1") + ": " + strings.limit(response.content.decode("latin1"), 100 if self.debug else 10000))
if self.debug:
Log.note("response: {{response}}", response=utf82unicode(response.content)[:130])
details = mo_json.json2value(utf82unicode(response.content))
if details.error:
Log.error(convert.quote2string(details.error))
if details._shards.failed > 0:
Log.error("Shard failures {{failures|indent}}",
failures="---\n".join(r.replace(";", ";\n") for r in details._shards.failures.reason)
)
return details
except Exception, e:
if url[0:4] != "http":
suggestion = " (did you forget \"http://\" prefix on the host name?)"
else:
suggestion = ""
if kwargs.get("data"):
Log.error(
"Problem with call to {{url}}" + suggestion + "\n{{body|left(10000)}}",
url=url,
body=strings.limit(kwargs["data"], 100 if self.debug else 10000),
cause=e
)
else:
Log.error("Problem with call to {{url}}" + suggestion, url=url, cause=e)
def delete_index(self, index_name):
if not isinstance(index_name, unicode):
Log.error("expecting an index name")
if self.debug:
Log.note("Deleting index {{index}}", index=index_name)
# REMOVE ALL ALIASES TOO
aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None]
if aliases:
self.post(
path="/_aliases",
data={"actions": [{"remove": a} for a in aliases]}
)
url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name
try:
response = http.delete(url)
if response.status_code != 200:
Log.error("Expecting a 200, got {{code}}", code=response.status_code)
details = mo_json.json2value(utf82unicode(response.content))
if self.debug:
Log.note("delete response {{response}}", response=details)
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def delete_index(self, index_name):
if not isinstance(index_name, unicode):
Log.error("expecting an index name")
if self.debug:
Log.note("Deleting index {{index}}", index=index_name)
# REMOVE ALL ALIASES TOO
aliases = [a for a in self.get_aliases() if a.index == index_name and a.alias != None]
if aliases:
self.post(
path="/_aliases",
data={"actions": [{"remove": a} for a in aliases]}
)
url = self.settings.host + ":" + unicode(self.settings.port) + "/" + index_name
try:
response = http.delete(url)
if response.status_code != 200:
Log.error("Expecting a 200, got {{code}}", code=response.status_code)
details = mo_json.json2value(utf82unicode(response.content))
if self.debug:
Log.note("delete response {{response}}", response=details)
return response
except Exception as e:
Log.error("Problem with call to {{url}}", url=url, cause=e)
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = utf82unicode(k)
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def test_multiple_agg_on_same_field(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [{
"name": "max_bytes",
"value": "run.stats.bytes",
"aggregate": "max"
}, {
"name": "count",
"value": "run.stats.bytes",
"aggregate": "count"
}]
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def _dict2json(value, sub_schema, path, net_new_properties, buffer):
prefix = '{'
for k, v in sort_using_key(value.items(), lambda r: r[0]):
if v == None or v == '':
continue
append(buffer, prefix)
prefix = COMMA
if is_binary(k):
k = utf82unicode(k)
if not is_text(k):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path + [k])
append(buffer, encode_basestring(encode_property(k)))
append(buffer, COLON)
typed_encode(v, sub_schema[k], path + [k], net_new_properties, buffer)
if prefix is COMMA:
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
def post_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
if 'json' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['json']))
del kwargs['json']
elif 'data' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['data']))
else:
Log.error(u"Expecting `json` parameter")
response = post(url, **kwargs)
details = json2value(utf82unicode(response.content))
if response.status_code not in [200, 201, 202]:
if "template" in details:
Log.error(u"Bad response code {{code}}",
code=response.status_code,
cause=Except.wrap(details))
else:
Log.error(u"Bad response code {{code}}\n{{details}}",
code=response.status_code,
details=details)
else:
return details
def test_timing(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [{
"name": "count",
"value": "run.duration",
"aggregate": "count"
}, {
"name": "total",
"value": "run.duration",
"aggregate": "sum"
}],
"edges": [{
"name": "chunk",
"value": ["run.suite", "run.chunk"]
}, "result.ok"],
"where": {
"and": [{
"lt": {
"timestamp": Date.floor(Date.now()).milli / 1000
}
}, {
"gte": {
"timestamp":
Date.floor(Date.now() - (Duration.DAY * 7),
Duration.DAY).milli / 1000
}
}]
},
"format":
"cube",
"samples": {
"limit": 30
}
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def error(response):
response = utf82unicode(response.content)
try:
e = Except.new_instance(json2value(response))
except Exception:
e = None
if e:
Log.error("Failed request", e)
else:
Log.error("Failed request\n {{response}}", {"response": response})
def test_simple_query(self):
if self.not_real_service():
return
query = unicode2utf8(convert.value2json({"from": "unittest"}))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def get_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
response = get(url, **kwargs)
try:
c = response.all_content
return json2value(utf82unicode(c))
except Exception as e:
if mo_math.round(response.status_code, decimal=-2) in [400, 500]:
Log.error(u"Bad GET response: {{code}}", code=response.status_code)
else:
Log.error(u"Good GET requests, but bad JSON", cause=e)
def get_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
response = get(url, **kwargs)
try:
c = response.all_content
return json2value(utf82unicode(c))
except Exception as e:
if Math.round(response.status_code, decimal=-2) in [400, 500]:
Log.error(u"Bad GET response: {{code}}", code=response.status_code)
else:
Log.error(u"Good GET requests, but bad JSON", cause=e)
def _dict2json(value, _buffer):
prefix = u'{"$object": ".", "'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
if not isinstance(k, unicode):
Log.error("Expecting property name to be a string")
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_typed_encode(v, _buffer)
append(_buffer, u"}")
def _dict2json(value, _buffer):
prefix = u'{"$object": ".", "'
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
if not isinstance(k, unicode):
Log.error("Expecting property name to be a string")
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_typed_encode(v, _buffer)
append(_buffer, u"}")
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key=key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return LazyLines(source)
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key=key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return utf82unicode(source.read()).split("\n")
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
return LazyLines(source)
def set_refresh_interval(self, seconds, **kwargs):
"""
:param seconds: -1 FOR NO REFRESH
:param kwargs: ANY OTHER REQUEST PARAMETERS
:return: None
"""
if seconds <= 0:
interval = -1
else:
interval = unicode(seconds) + "s"
if self.cluster.version.startswith("0.90."):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}',
**kwargs
)
result = mo_json.json2value(utf82unicode(response.all_content))
if not result.ok:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'),
**kwargs
)
result = mo_json.json2value(utf82unicode(response.all_content))
if not result.acknowledged:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
else:
Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
def set_refresh_interval(self, seconds, **kwargs):
"""
:param seconds: -1 FOR NO REFRESH
:param kwargs: ANY OTHER REQUEST PARAMETERS
:return: None
"""
if seconds <= 0:
interval = -1
else:
interval = unicode(seconds) + "s"
if self.cluster.version.startswith("0.90."):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data='{"index":{"refresh_interval":' + convert.value2json(interval) + '}}',
**kwargs
)
result = mo_json.json2value(utf82unicode(response.all_content))
if not result.ok:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
elif any(map(self.cluster.version.startswith, ["1.4.", "1.5.", "1.6.", "1.7."])):
response = self.cluster.put(
"/" + self.settings.index + "/_settings",
data=convert.unicode2utf8('{"index":{"refresh_interval":' + convert.value2json(interval) + '}}'),
**kwargs
)
result = mo_json.json2value(utf82unicode(response.all_content))
if not result.acknowledged:
Log.error("Can not set refresh interval ({{error}})", {
"error": utf82unicode(response.all_content)
})
else:
Log.error("Do not know how to handle ES version {{version}}", version=self.cluster.version)
def put(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
if self.debug:
sample = kwargs["data"][:300]
Log.note("PUT {{url}}:\n{{data|indent}}", url=url, data=sample)
try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:])
return response
except Exception, e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def put(self, path, **kwargs):
url = self.settings.host + ":" + unicode(self.settings.port) + path
if self.debug:
sample = kwargs["data"][:300]
Log.note("PUT {{url}}:\n{{data|indent}}", url=url, data=sample)
try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason+": "+response.all_content)
if self.debug:
Log.note("response: {{response}}", response= utf82unicode(response.all_content)[0:300:])
return response
except Exception as e:
Log.error("Problem with call to {{url}}", url= url, cause=e)
def get(*args, **kwargs):
body = kwargs.get("data")
if not body:
return wrap({"status_code": 400})
text = utf82unicode(body)
data = json2value(text)
result = jx.run(data)
output_bytes = unicode2utf8(value2json(result))
return wrap({
"status_code": 200,
"all_content": output_bytes,
"content": output_bytes
})
def execute_query(self, query):
query = wrap(query)
try:
query = unicode2utf8(value2json(query))
# EXECUTE QUERY
response = self.try_till_response(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
return result
except Exception as e:
Log.error("Failed query", e)
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return utf82unicode(json)
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return utf82unicode(json)
def annotation():
if flask.request.headers.get("content-length", "") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(ERROR_CONTENT,
status=400,
headers={"Content-Type": "text/html"})
elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT:
Log.error("Query is too large to parse")
request_body = flask.request.get_data().strip()
text = utf82unicode(request_body)
data = json2value(text)
try:
record_request(flask.request, data, None, None)
except Exception as e:
Log.error("Problem processing request {{request}}")
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.items():
append(_buffer, prefix)
prefix = COMMA_QUOTE
if is_binary(k):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE_COLON)
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _dict2json(value, _buffer):
try:
prefix = u"{\""
for k, v in value.iteritems():
append(_buffer, prefix)
prefix = u", \""
if isinstance(k, str):
k = utf82unicode(k)
for c in k:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u"\": ")
_value2json(v, _buffer)
append(_buffer, u"}")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", cause=e)
def send_queries(self, subtest, places=6):
subtest = wrap(subtest)
try:
# EXECUTE QUERY
num_expectations = 0
for i, (k, v) in enumerate(subtest.items()):
if k.startswith("expecting_"): # WHAT FORMAT ARE WE REQUESTING
format = k[len("expecting_"):]
elif k == "expecting": # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS)
format = None
else:
continue
num_expectations += 1
expected = v
subtest.query.format = format
subtest.query.meta.testing = (
num_expectations == 1
) # MARK FIRST QUERY FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION
query = unicode2utf8(value2json(subtest.query))
# EXECUTE QUERY
response = self.try_till_response(self.testing.query,
data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
container = jx_elasticsearch.new_instance(
self._es_test_settings)
query = QueryOp.wrap(subtest.query, container,
container.namespace)
compare_to_expected(query, result, expected, places)
Log.note("PASS {{name|quote}} (format={{format}})",
name=subtest.name,
format=format)
if num_expectations == 0:
Log.error(
"Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses",
name=subtest.name)
except Exception as e:
Log.error("Failed test {{name|quote}}", {"name": subtest.name}, e)
def test_longest_running_tests(self):
test = wrap({
"query": {
"sort": {
"sort": -1,
"field": "avg"
},
"from": {
"from":
"unittest",
"where": {
"and": [{
"gt": {
"build.date": "1439337600"
}
}]
},
"groupby": [
"build.platform", "build.type", "run.suite",
"result.test"
],
"select": [{
"aggregate": "avg",
"name": "avg",
"value": "result.duration"
}],
"format":
"table",
"limit":
100
},
"limit": 100,
"format": "list"
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def test_branch_count(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [
{
"aggregate": "count"
},
],
"edges": ["build.branch"],
"where": {
"or":
[{
"missing": "build.id"
}
# {"gte": {"timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000}}
]
},
"format": "table"
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def _dict2json(self, value, sub_schema, path, net_new_properties, _buffer):
prefix = '{'
for k, v in ((kk, value[kk]) for kk in sorted(value.keys())):
if v == None or v == '':
continue
append(_buffer, prefix)
prefix = COMMA
if isinstance(k, binary_type):
k = utf82unicode(k)
if not isinstance(k, text_type):
Log.error("Expecting property name to be a string")
if k not in sub_schema:
sub_schema[k] = {}
net_new_properties.append(path+[k])
append(_buffer, json.dumps(encode_property(k)))
append(_buffer, COLON)
self._typed_encode(v, sub_schema[k], path+[k], net_new_properties, _buffer)
if prefix == COMMA:
append(_buffer, COMMA+QUOTED_EXISTS_TYPE+COLON+'1}')
else:
append(_buffer, '{'+QUOTED_EXISTS_TYPE+COLON+'0}')
def post_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
if 'json' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['json']))
del kwargs['json']
elif 'data' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['data']))
else:
Log.error(u"Expecting `json` parameter")
response = post(url, **kwargs)
details = json2value(utf82unicode(response.content))
if response.status_code not in [200, 201, 202]:
if "template" in details:
Log.error(u"Bad response code {{code}}", code=response.status_code, cause=Except.wrap(details))
else:
Log.error(u"Bad response code {{code}}\n{{details}}", code=response.status_code, details=details)
else:
return details
def post_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
if 'json' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['json']))
elif 'data' in kwargs:
kwargs['data'] = unicode2utf8(value2json(kwargs['data']))
else:
Log.error(u"Expecting `json` parameter")
response = post(url, **kwargs)
c = response.content
try:
details = json2value(utf82unicode(c))
except Exception as e:
Log.error(u"Unexpected return value {{content}}", content=c, cause=e)
if response.status_code not in [200, 201]:
Log.error(u"Bad response", cause=Except.wrap(details))
return details
def test_failures_by_directory(self):
if self.not_real_service():
return
test = wrap({
"query": {
"from": {
"type": "elasticsearch",
"settings": {
"host": ES_CLUSTER_LOCATION,
"index": "unittest",
"type": "test_result"
}
},
"select": [{
"aggregate": "count"
}],
"edges": ["result.test", "result.ok"],
"where": {
"prefix": {
"result.test": "/"
}
},
"format": "table"
}
})
query = unicode2utf8(convert.value2json(test.query))
# EXECUTE QUERY
with Timer("query"):
response = http.get(self.testing.query, data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
Log.note("result\n{{result|indent}}", {"result": result})
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif isinstance(value, Mapping):
try:
items = sort_using_key(list(value.items()), lambda r: r[0])
values = [
encode_basestring(k) + PRETTY_COLON +
indent(pretty_json(v)).strip() for k, v in items
if v != None
]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + INDENT + (",\n" +
INDENT).join(values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not isinstance(k, text_type) for k in value.keys()):
Log.error("JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e)
Log.error("problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e)
elif value in (None, Null):
return "null"
elif isinstance(value, (text_type, binary_type)):
if isinstance(value, binary_type):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note(
"try explicit convert of string with length {{length}}",
length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text_type(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}",
length=len(output))
return output
except BaseException as f:
Log.warning("can not even explicit convert {{type}}",
type=f.__class__.__name__,
cause=f)
return "null"
elif isinstance(value, list):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(
*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(
1,
min(
ARRAY_MAX_COLUMNS,
int(
floor((ARRAY_ROW_LENGTH + 2.0) /
float(max_len +
2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join(
[indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(
j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns))
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning(
"problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p))
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text_type(int(value))
except Exception:
pass
try:
if float(value) == value:
return text_type(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def _value2json(value, _buffer):
try:
_class = value.__class__
if value is None:
append(_buffer, u"null")
return
elif value is True:
append(_buffer, u"true")
return
elif value is False:
append(_buffer, u"false")
return
type = value.__class__
if type is binary_type:
append(_buffer, QUOTE)
try:
v = utf82unicode(value)
except Exception as e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is text_type:
append(_buffer, QUOTE)
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is dict:
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif type is Data:
d = _get(value, "_dict") # MIGHT BE A VALUE NOT A DICT
_value2json(d, _buffer)
return
elif type in (int, long, Decimal):
append(_buffer, text_type(value))
elif type is float:
if math.isnan(value) or math.isinf(value):
append(_buffer, u'null')
else:
append(_buffer, float2json(value))
elif type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif type is date:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is datetime:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is Date:
append(_buffer, float2json(value.unix))
elif type is timedelta:
append(_buffer, float2json(value.total_seconds()))
elif type is Duration:
append(_buffer, float2json(value.seconds))
elif type is NullType:
append(_buffer, u"null")
elif isinstance(value, Mapping):
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif hasattr(value, '__data__'):
d = value.__data__()
_value2json(d, _buffer)
elif hasattr(value, '__json__'):
j = value.__json__()
append(_buffer, j)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable",
cause=e)
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif is_data(value):
try:
items = sort_using_key(value.items(), lambda r: r[0])
values = [encode_basestring(k) + PRETTY_COLON + pretty_json(v) for k, v in items if v != None]
if not values:
return "{}"
elif len(values) == 1:
return "{" + values[0] + "}"
else:
return "{\n" + ",\n".join(indent(v) for v in values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not is_text(k) for k in value.keys()):
Log.error(
"JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
Log.error(
"problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
elif value in (None, Null):
return "null"
elif value.__class__ in (binary_type, text_type):
if is_binary(value):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = [QUOTE]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = text_type(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(QUOTE)
output = u"".join(acc)
Log.note("return value of length {{length}}", length=len(output))
return output
except BaseException as f:
Log.warning("can not convert {{type}} to json", type=f.__class__.__name__, cause=f)
return "null"
elif is_list(value):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + PRETTY_COMMA.join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
PRETTY_COMMA.join(j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns)
)
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning("problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p)
)
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
elif hasattr(value, '__data__'):
d = value.__data__()
return pretty_json(d)
elif hasattr(value, '__json__'):
j = value.__json__()
if j == None:
return " null " # TODO: FIND OUT WHAT CAUSES THIS
return pretty_json(json_decoder(j))
elif scrub(value) is None:
return "null"
elif hasattr(value, '__iter__'):
return pretty_json(list(value))
elif hasattr(value, '__call__'):
return "null"
else:
try:
if int(value) == value:
return text_type(int(value))
except Exception:
pass
try:
if float(value) == value:
return text_type(float(value))
except Exception:
pass
return pypy_json_encode(value)
except Exception as e:
problem_serializing(value, e)
def pretty_json(value):
try:
if value is False:
return "false"
elif value is True:
return "true"
elif isinstance(value, Mapping):
try:
if not value:
return "{}"
items = list(value.items())
if len(items) == 1:
return "{" + unicode_key(items[0][0]) + ": " + pretty_json(items[0][1]).strip() + "}"
items = sorted(items, lambda a, b: value_compare(a[0], b[0]))
values = [unicode_key(k) + ": " + indent(pretty_json(v)).strip() for k, v in items if v != None]
return "{\n" + INDENT + (",\n" + INDENT).join(values) + "\n}"
except Exception as e:
from mo_logs import Log
from mo_math import OR
if OR(not isinstance(k, basestring) for k in value.keys()):
Log.error(
"JSON must have string keys: {{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
Log.error(
"problem making dict pretty: keys={{keys}}:",
keys=[k for k in value.keys()],
cause=e
)
elif value in (None, Null):
return "null"
elif isinstance(value, basestring):
if isinstance(value, str):
value = utf82unicode(value)
try:
return quote(value)
except Exception as e:
from mo_logs import Log
try:
Log.note("try explicit convert of string with length {{length}}", length=len(value))
acc = [u"\""]
for c in value:
try:
try:
c2 = ESCAPE_DCT[c]
except Exception:
c2 = c
c3 = unicode(c2)
acc.append(c3)
except BaseException:
pass
# Log.warning("odd character {{ord}} found in string. Ignored.", ord= ord(c)}, cause=g)
acc.append(u"\"")
output = u"".join(acc)
Log.note("return value of length {{length}}", length=len(output))
return output
except BaseException, f:
Log.warning("can not even explicit convert {{type}}", type=f.__class__.__name__, cause=f)
return "null"
elif isinstance(value, list):
if not value:
return "[]"
if ARRAY_MAX_COLUMNS == 1:
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
if len(value) == 1:
j = pretty_json(value[0])
if j.find("\n") >= 0:
return "[\n" + indent(j) + "\n]"
else:
return "[" + j + "]"
js = [pretty_json(v) for v in value]
max_len = max(*[len(j) for j in js])
if max_len <= ARRAY_ITEM_MAX_LENGTH and max(*[j.find("\n") for j in js]) == -1:
# ALL TINY VALUES
num_columns = max(1, min(ARRAY_MAX_COLUMNS, int(floor((ARRAY_ROW_LENGTH + 2.0) / float(max_len + 2))))) # +2 TO COMPENSATE FOR COMMAS
if len(js) <= num_columns: # DO NOT ADD \n IF ONLY ONE ROW
return "[" + ", ".join(js) + "]"
if num_columns == 1: # DO NOT rjust IF THERE IS ONLY ONE COLUMN
return "[\n" + ",\n".join([indent(pretty_json(v)) for v in value]) + "\n]"
content = ",\n".join(
", ".join(j.rjust(max_len) for j in js[r:r + num_columns])
for r in xrange(0, len(js), num_columns)
)
return "[\n" + indent(content) + "\n]"
pretty_list = js
output = ["[\n"]
for i, p in enumerate(pretty_list):
try:
if i > 0:
output.append(",\n")
output.append(indent(p))
except Exception:
from mo_logs import Log
Log.warning("problem concatenating string of length {{len1}} and {{len2}}",
len1=len("".join(output)),
len2=len(p)
)
output.append("\n]")
try:
return "".join(output)
except Exception as e:
from mo_logs import Log
Log.error("not expected", cause=e)
def _value2json(value, _buffer):
try:
_class = value.__class__
if value is None:
append(_buffer, u"null")
return
elif value is True:
append(_buffer, u"true")
return
elif value is False:
append(_buffer, u"false")
return
type = value.__class__
if type is binary_type:
append(_buffer, QUOTE)
try:
v = utf82unicode(value)
except Exception as e:
problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is text_type:
append(_buffer, QUOTE)
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, QUOTE)
elif type is dict:
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif type is Data:
d = _get(value, SLOT) # MIGHT BE A VALUE NOT A DICT
_value2json(d, _buffer)
return
elif type in (int, long, Decimal):
append(_buffer, text_type(value))
elif type is float:
if math.isnan(value) or math.isinf(value):
append(_buffer, u'null')
else:
append(_buffer, float2json(value))
elif type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif type is date:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is datetime:
append(_buffer, float2json(time.mktime(value.timetuple())))
elif type is Date:
append(_buffer, float2json(value.unix))
elif type is timedelta:
append(_buffer, float2json(value.total_seconds()))
elif type is Duration:
append(_buffer, float2json(value.seconds))
elif type is NullType:
append(_buffer, u"null")
elif is_data(value):
if not value:
append(_buffer, u"{}")
else:
_dict2json(value, _buffer)
return
elif hasattr(value, '__data__'):
d = value.__data__()
_value2json(d, _buffer)
elif hasattr(value, '__json__'):
j = value.__json__()
append(_buffer, j)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def _typed_encode(value, _buffer):
try:
if value is None:
append(_buffer, u'{"$value": null}')
return
elif value is True:
append(_buffer, u'{"$value": true}')
return
elif value is False:
append(_buffer, u'{"$value": false}')
return
_type = value.__class__
if _type in (dict, Data):
if value:
_dict2json(value, _buffer)
else:
append(_buffer, u'{"$object": "."}')
elif _type is str:
append(_buffer, u'{"$value": "')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type is unicode:
append(_buffer, u'{"$value": "')
for c in value:
append(_buffer, ESCAPE_DCT.get(c, c))
append(_buffer, u'"}')
elif _type in (int, long, Decimal):
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type is float:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value))
append(_buffer, u'}')
elif _type in (set, list, tuple, FlatList):
_list2json(value, _buffer)
elif _type is date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is datetime:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.timetuple())))
append(_buffer, u'}')
elif _type is Date:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(time.mktime(value.value.timetuple())))
append(_buffer, u'}')
elif _type is timedelta:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.total_seconds()))
append(_buffer, u'}')
elif _type is Duration:
append(_buffer, u'{"$value": ')
append(_buffer, float2json(value.seconds))
append(_buffer, u'}')
elif _type is NullType:
append(_buffer, u"null")
elif hasattr(value, '__json__'):
j = value.__json__()
t = json2typed(j)
append(_buffer, t)
elif hasattr(value, '__iter__'):
_iter2json(value, _buffer)
else:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(_repr(value) + " is not JSON serializable", e)
def typed_encode(value, sub_schema, path, net_new_properties, buffer):
"""
:param value: THE DATA STRUCTURE TO ENCODE
:param sub_schema: dict FROM PATH TO Column DESCRIBING THE TYPE
:param path: list OF CURRENT PATH
:param net_new_properties: list FOR ADDING NEW PROPERTIES NOT FOUND IN sub_schema
:param buffer: UnicodeBuilder OBJECT
:return:
"""
try:
# from jx_base import Column
if sub_schema.__class__.__name__=='Column':
value_json_type = python_type_to_json_type[value.__class__]
column_json_type = es_type_to_json_type[sub_schema.es_type]
if value_json_type == column_json_type:
pass # ok
elif value_json_type == NESTED and all(python_type_to_json_type[v.__class__] == column_json_type for v in value if v != None):
pass # empty arrays can be anything
else:
from mo_logs import Log
Log.error("Can not store {{value}} in {{column|quote}}", value=value, column=sub_schema.name)
sub_schema = {json_type_to_inserter_type[value_json_type]: sub_schema}
if value == None:
from mo_logs import Log
Log.error("can not encode null (missing) values")
elif value is True:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'true}')
return
elif value is False:
if BOOLEAN_TYPE not in sub_schema:
sub_schema[BOOLEAN_TYPE] = {}
net_new_properties.append(path + [BOOLEAN_TYPE])
append(buffer, '{')
append(buffer, QUOTED_BOOLEAN_TYPE)
append(buffer, 'false}')
return
_type = value.__class__
if _type in (dict, Data):
if sub_schema.__class__.__name__ == 'Column':
from mo_logs import Log
Log.error("Can not handle {{column|json}}", column=sub_schema)
if NESTED_TYPE in sub_schema:
# PREFER NESTED, WHEN SEEN BEFORE
if value:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[')
_dict2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, ']' + COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, text_type(len(value)))
append(buffer, '}')
else:
# SINGLETON LIST
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}]')
append(buffer, COMMA)
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
else:
if EXISTS_TYPE not in sub_schema:
sub_schema[EXISTS_TYPE] = {}
net_new_properties.append(path + [EXISTS_TYPE])
if value:
_dict2json(value, sub_schema, path, net_new_properties, buffer)
else:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '1}')
elif _type is binary_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
try:
v = utf82unicode(value)
except Exception as e:
raise problem_serializing(value, e)
for c in v:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type is text_type:
if STRING_TYPE not in sub_schema:
sub_schema[STRING_TYPE] = True
net_new_properties.append(path + [STRING_TYPE])
append(buffer, '{')
append(buffer, QUOTED_STRING_TYPE)
append(buffer, '"')
for c in value:
append(buffer, ESCAPE_DCT.get(c, c))
append(buffer, '"}')
elif _type in (int, long):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, text_type(value))
append(buffer, '}')
elif _type in (float, Decimal):
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value))
append(buffer, '}')
elif _type in (set, list, tuple, FlatList):
if len(value) == 0:
append(buffer, '{')
append(buffer, QUOTED_EXISTS_TYPE)
append(buffer, '0}')
elif any(v.__class__ in (Data, dict, set, list, tuple, FlatList) for v in value):
# THIS IS NOT DONE BECAUSE
if len(value) == 1:
if NESTED_TYPE in sub_schema:
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
# NO NEED TO NEST, SO DO NOT DO IT
typed_encode(value[0], sub_schema, path, net_new_properties, buffer)
else:
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_list2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
# ALLOW PRIMITIVE MULTIVALUES
value = [v for v in value if v != None]
types = list(set(json_type_to_inserter_type[python_type_to_json_type[v.__class__]] for v in value))
if len(types) == 0: # HANDLE LISTS WITH Nones IN THEM
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
append(buffer, '[]}')
elif len(types) > 1:
_list2json(value, sub_schema, path + [NESTED_TYPE], net_new_properties, buffer)
else:
element_type = types[0]
if element_type not in sub_schema:
sub_schema[element_type] = True
net_new_properties.append(path + [element_type])
append(buffer, '{')
append(buffer, quote(element_type))
append(buffer, COLON)
_multivalue2json(value, sub_schema[element_type], path + [element_type], net_new_properties, buffer)
append(buffer, '}')
elif _type is date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is datetime:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(time.mktime(value.timetuple())))
append(buffer, '}')
elif _type is Date:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.unix))
append(buffer, '}')
elif _type is timedelta:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.total_seconds()))
append(buffer, '}')
elif _type is Duration:
if NUMBER_TYPE not in sub_schema:
sub_schema[NUMBER_TYPE] = True
net_new_properties.append(path + [NUMBER_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NUMBER_TYPE)
append(buffer, float2json(value.seconds))
append(buffer, '}')
elif _type is NullType:
append(buffer, 'null')
elif hasattr(value, '__data__'):
typed_encode(value.__data__(), sub_schema, path, net_new_properties, buffer)
elif hasattr(value, '__iter__'):
if NESTED_TYPE not in sub_schema:
sub_schema[NESTED_TYPE] = {}
net_new_properties.append(path + [NESTED_TYPE])
append(buffer, '{')
append(buffer, QUOTED_NESTED_TYPE)
_iter2json(value, sub_schema[NESTED_TYPE], path + [NESTED_TYPE], net_new_properties, buffer)
append(buffer, '}')
else:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable")
except Exception as e:
from mo_logs import Log
Log.error(text_type(repr(value)) + " is not JSON serializable", cause=e)
def tuid_endpoint(path):
try:
if flask.request.headers.get("content-length", "") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
return Response(EXPECTING_QUERY,
status=400,
headers={"Content-Type": "text/html"})
elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT:
return Response(unicode2utf8("request too large"),
status=400,
headers={"Content-Type": "text/html"})
request_body = flask.request.get_data().strip()
query = json2value(utf82unicode(request_body))
# ENSURE THE QUERY HAS THE CORRECT FORM
if query['from'] != 'files':
Log.error("Can only handle queries on the `files` table")
ands = listwrap(query.where['and'])
if len(ands) != 3:
Log.error(
'expecting a simple where clause with following structure\n{{example|json}}',
example={
"and": [{
"eq": {
"branch": "<BRANCH>"
}
}, {
"eq": {
"revision": "<REVISION>"
}
}, {
"in": {
"path": ["<path1>", "<path2>", "...", "<pathN>"]
}
}]
})
rev = None
paths = None
branch_name = None
for a in ands:
rev = coalesce(rev, a.eq.revision)
paths = unwraplist(coalesce(paths, a['in'].path, a.eq.path))
branch_name = coalesce(branch_name, a.eq.branch)
paths = listwrap(paths)
if len(paths) == 0:
response, completed = [], True
elif service.conn.pending_transactions > TOO_BUSY: # CHECK IF service IS VERY BUSY
# TODO: BE SURE TO UPDATE STATS TOO
Log.note("Too many open transactions")
response, completed = [], False
else:
# RETURN TUIDS
with Timer("tuid internal response time for {{num}} files",
{"num": len(paths)}):
response, completed = service.get_tuids_from_files(
revision=rev,
files=paths,
going_forward=True,
repo=branch_name)
if not completed:
Log.note(
"Request for {{num}} files is incomplete for revision {{rev}}.",
num=len(paths),
rev=rev)
if query.meta.format == 'list':
formatter = _stream_list
else:
formatter = _stream_table
return Response(formatter(response),
status=200 if completed else 202,
headers={"Content-Type": mimetype.JSON})
except Exception as e:
e = Except.wrap(e)
Log.warning("could not handle request", cause=e)
return Response(unicode2utf8(value2json(e, pretty=True)),
status=400,
headers={"Content-Type": "text/html"})
def tuid_endpoint(path):
with RegisterThread():
try:
service.statsdaemon.update_requests(requests_total=1)
if flask.request.headers.get("content-length", "") in ["", "0"]:
# ASSUME A BROWSER HIT THIS POINT, SEND text/html RESPONSE BACK
service.statsdaemon.update_requests(requests_complete=1, requests_passed=1)
return Response(
EXPECTING_QUERY,
status=400,
headers={
"Content-Type": "text/html"
}
)
elif int(flask.request.headers["content-length"]) > QUERY_SIZE_LIMIT:
service.statsdaemon.update_requests(requests_complete=1, requests_passed=1)
return Response(
unicode2utf8("request too large"),
status=400,
headers={
"Content-Type": "text/html"
}
)
request_body = flask.request.get_data().strip()
query = json2value(utf82unicode(request_body))
# ENSURE THE QUERY HAS THE CORRECT FORM
if query['from'] != 'files':
Log.error("Can only handle queries on the `files` table")
ands = listwrap(query.where['and'])
if len(ands) != 3:
Log.error(
'expecting a simple where clause with following structure\n{{example|json}}',
example={"and": [
{"eq": {"branch": "<BRANCH>"}},
{"eq": {"revision": "<REVISION>"}},
{"in": {"path": ["<path1>", "<path2>", "...", "<pathN>"]}}
]}
)
rev = None
paths = None
branch_name = None
for a in ands:
rev = coalesce(rev, a.eq.revision)
paths = unwraplist(coalesce(paths, a['in'].path, a.eq.path))
branch_name = coalesce(branch_name, a.eq.branch)
paths = listwrap(paths)
if len(paths) == 0:
response, completed = [], True
elif service.conn.pending_transactions > TOO_BUSY: # CHECK IF service IS VERY BUSY
# TODO: BE SURE TO UPDATE STATS TOO
Log.note("Too many open transactions")
response, completed = [], False
elif service.get_thread_count() > TOO_MANY_THREADS:
Log.note("Too many threads open")
response, completed = [], False
else:
# RETURN TUIDS
with Timer("tuid internal response time for {{num}} files", {"num": len(paths)}):
response, completed = service.get_tuids_from_files(
revision=rev, files=paths, going_forward=True, repo=branch_name
)
if not completed:
Log.note(
"Request for {{num}} files is incomplete for revision {{rev}}.",
num=len(paths), rev=rev
)
if query.meta.format == 'list':
formatter = _stream_list
else:
formatter = _stream_table
service.statsdaemon.update_requests(
requests_complete=1 if completed else 0,
requests_incomplete=1 if not completed else 0,
requests_passed=1
)
return Response(
formatter(response),
status=200 if completed else 202,
headers={
"Content-Type": "application/json"
}
)
except Exception as e:
e = Except.wrap(e)
service.statsdaemon.update_requests(requests_incomplete=1, requests_failed=1)
Log.warning("could not handle request", cause=e)
return Response(
unicode2utf8(value2json(e, pretty=True)),
status=400,
headers={
"Content-Type": "text/html"
}
)
