def solve():
try:
data = convert.json2value(convert.utf82unicode(flask.request.data))
solved = noop.solve(data)
response_data = convert.unicode2utf8(convert.value2json(solved))
return Response(
response_data,
direct_passthrough=True, # FOR STREAMING
status=200,
headers={
"access-control-allow-origin": "*",
"content-type": "application/json"
}
)
except Exception, e:
e = Except.wrap(e)
Log.warning("Could not process", cause=e)
e = e.as_dict()
return Response(
convert.unicode2utf8(convert.value2json(e)),
status=400,
headers={
"access-control-allow-origin": "*",
"content-type": "application/json"
}
)
def test_big_result_works(self):
result = http.post_json(global_settings.service_url,
data={
"from": "unittest",
"where": {
"and": [{
"gte": {
"run.timestamp":
Date.today() - DAY
}
}, {
"lt": {
"run.timestamp": Date.today()
}
}, {
"eq": {
"result.ok": False
}
}]
},
"format": "list",
"limit": 10000
})
if result.template:
result = Except.new_instance(result)
Log.error("problem with call", cause=result)
Log.note("Got {{num}} test failures", num=len(result.data))
def json2value(json_string, params={}, flexible=False, leaves=False):
"""
:param json_string: THE JSON
:param params: STANDARD JSON PARAMS
:param flexible: REMOVE COMMENTS
:param leaves: ASSUME JSON KEYS ARE DOT-DELIMITED
:return: Python value
"""
if isinstance(json_string, str):
Log.error("only unicode json accepted")
try:
if flexible:
# REMOVE """COMMENTS""", # COMMENTS, //COMMENTS, AND \n \r
# DERIVED FROM https://github.com/jeads/datasource/blob/master/datasource/bases/BaseHub.py# L58
json_string = re.sub(r"\"\"\".*?\"\"\"", r"\n", json_string, flags=re.MULTILINE)
json_string = "\n".join(remove_line_comment(l) for l in json_string.split("\n"))
# ALLOW DICTIONARY'S NAME:VALUE LIST TO END WITH COMMA
json_string = re.sub(r",\s*\}", r"}", json_string)
# ALLOW LISTS TO END WITH COMMA
json_string = re.sub(r",\s*\]", r"]", json_string)
if params:
json_string = expand_template(json_string, params)
# LOOKUP REFERENCES
value = wrap(json_decoder(json_string))
if leaves:
value = wrap_leaves(value)
return value
except Exception, e:
e = Except.wrap(e)
if "Expecting '" in e and "' delimiter: line" in e:
line_index = int(strings.between(e.message, " line ", " column ")) - 1
column = int(strings.between(e.message, " column ", " ")) - 1
line = json_string.split("\n")[line_index].replace("\t", " ")
if column > 20:
sample = "..." + line[column - 20:]
pointer = " " + (" " * 20) + "^"
else:
sample = line
pointer = (" " * column) + "^"
if len(sample) > 43:
sample = sample[:43] + "..."
Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n")
base_str = unicode2utf8(strings.limit(json_string, 1000))
hexx_str = bytes2hex(base_str, " ")
try:
char_str = " " + (" ".join(c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str)
except Exception:
char_str = " "
Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e)
def write(self, template, params):
try:
self.queue.add({"template": template, "params": params})
return self
except Exception, e:
e = _Except.wrap(e)
sys.stdout.write("IF YOU SEE THIS, IT IS LIKELY YOU FORGOT TO RUN Log.start() FIRST\n")
raise e # OH NO!
def _kill(self):
try:
self.service.kill()
except Exception, e:
ee = Except.wrap(e)
if "The operation completed successfully" in ee:
return
Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
def try_till_response(self, *args, **kwargs):
while True:
try:
response = self.server.get(*args, **kwargs)
return response
except Exception, e:
e = Except.wrap(e)
if "No connection could be made because the target machine actively refused it" in e:
Log.alert("Problem connecting")
else:
Log.error("Server raised exception", e)
def error(response):
response = convert.utf82unicode(response.content)
try:
e = Except.new_instance(convert.json2value(response))
except Exception:
e = None
if e:
Log.error("Failed request", e)
else:
Log.error("Failed request\n {{response}}", {"response": response})
def test_trace_of_simple_raises(self):
try:
problem_a()
except Exception, e:
f = Except.wrap(e)
self.assertEqual(f.template, "expected exception")
for i, m in enumerate(listwrap(f.trace).method):
if m == "test_trace_of_simple_raises":
self.assertEqual(i, 2)
break
else:
self.fail("expecting stack to show this method")
def encode(self, value, pretty=False):
if pretty:
return pretty_json(value)
try:
scrubbed = scrub(value)
return unicode(self.encoder.encode(scrubbed))
except Exception, e:
from pyLibrary.debugs.logs import Log, Except
e = Except.wrap(e)
Log.warning("problem serializing {{type}}", type=_repr(value), cause=e)
raise e
def test_full_trace_on_wrap(self):
try:
problem_b()
except Exception, e:
cause = Except.wrap(e)
self.assertEqual(cause.template, "expected exception")
for i, m in enumerate(listwrap(cause.trace).method):
if m == "test_full_trace_on_wrap":
self.assertEqual(i, 1)
break
else:
self.fail("expecting stack to show this method")
def test_contains_from_zip_error(self):
def bad_unzip():
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
return decompressor.decompress(b'invlaid zip file')
try:
bad_unzip()
assert False
except Exception, e:
e = Except.wrap(e)
if "incorrect header check" in e:
pass
else:
assert False
def encode(self, value, pretty=False):
if pretty:
return pretty_json(value)
try:
scrubbed = scrub(value)
return unicode(self.encoder.encode(scrubbed))
except Exception, e:
from pyLibrary.debugs.logs import Log, Except
e = Except.wrap(e)
Log.warning("problem serializing {{type}}",
type=_repr(value),
cause=e)
raise e
def _got_result(self, data, message):
data = wrap(data)
data._meta.count = self.count
self.count += 1
if self.settings.debug:
Log.note("{{data}}", data=data)
if self.target_queue != None:
try:
self.target_queue.add(data)
message.ack()
except Exception, e:
e = Except.wrap(e)
if not self.target_queue.closed: # EXPECTED TO HAPPEN, THIS THREAD MAY HAVE BEEN AWAY FOR A WHILE
raise e
def _got_result(self, data, message):
data = wrap(data)
data._meta.count = self.count
self.count += 1
if self.settings.debug:
Log.note("{{data}}", data= data)
if self.target_queue != None:
try:
self.target_queue.add(data)
message.ack()
except Exception, e:
e = Except.wrap(e)
if not self.target_queue.closed: # EXPECTED TO HAPPEN, THIS THREAD MAY HAVE BEEN AWAY FOR A WHILE
raise e
def value2json(obj, pretty=False):
try:
json = json_encoder(obj, pretty=pretty)
if json == None:
Log.note(str(type(obj)) + " is not valid{{type}}JSON", type= " (pretty) " if pretty else " ")
Log.error("Not valid JSON: " + str(obj) + " of type " + str(type(obj)))
return json
except Exception, e:
e = Except.wrap(e)
try:
json = pypy_json_encode(obj)
return json
except Exception:
pass
Log.error("Can not encode into JSON: {{value}}", value=repr(obj), cause=e)
def utf82unicode(value):
"""
WITH EXPLANATION FOR FAILURE
"""
try:
return value.decode("utf8")
except Exception, e:
if not _Log:
_late_import()
if not isinstance(value, basestring):
_Log.error("Can not _convert {{type}} to unicode because it's not a string", type= type(value).__name__)
e = _Except.wrap(e)
for i, c in enumerate(value):
try:
c.decode("utf8")
except Exception, f:
_Log.error("Can not _convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])
def query(self, _query):
try:
if not self.ready:
Log.error("Must use with clause for any instance of FromES")
query = Query(_query, schema=self)
# try:
# frum = self.get_columns(query["from"])
# mvel = _MVEL(frum)
# except Exception, e:
# mvel = None
# Log.warning("TODO: Fix this", e)
#
for s in listwrap(query.select):
if not aggregates1_4[s.aggregate]:
Log.error("ES can not aggregate " + self.select[0].name +
" because '" + self.select[0].aggregate +
"' is not a recognized aggregate")
frum = query["from"]
if isinstance(frum, Query):
result = self.query(frum)
q2 = query.copy()
q2.frum = result
return qb.run(q2)
if is_aggsop(self._es, query):
return es_aggsop(self._es, frum, query)
if is_fieldop(self._es, query):
return es_fieldop(self._es, query)
if is_setop(self._es, query):
return es_setop(self._es, query)
Log.error("Can not handle")
except Exception, e:
e = Except.wrap(e)
if "Data too large, data for" in e:
http.post(self._es.cluster.path + "/_cache/clear")
Log.error("Problem (Tried to clear Elasticsearch cache)", e)
Log.error("problem", e)
def request(method, url, zip=None, retry=None, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
Parameters
* zip - ZIP THE REQUEST BODY, IF BIG ENOUGH
* json - JSON-SERIALIZABLE STRUCTURE
* retry - {"times": x, "sleep": y} STRUCTURE
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
if not default_headers and not _warning_sent:
_warning_sent = True
Log.warning(
"The pyLibrary.env.http module was meant to add extra "
"default headers to all requests, specifically the 'Referer' "
"header with a URL to the project. Use the `pyLibrary.debug.constants.set()` "
"function to set `pyLibrary.env.http.default_headers`"
)
if isinstance(url, list):
# TRY MANY URLS
failures = []
for remaining, u in qb.countdown(url):
try:
response = request(method, u, zip=zip, retry=retry, **kwargs)
if Math.round(response.status_code, decimal=-2) not in [400, 500]:
return response
if not remaining:
return response
except Exception, e:
e = Except.wrap(e)
failures.append(e)
Log.error("Tried {{num}} urls", num=len(url), cause=failures)
def output(*args):
with cache_store.locker:
if using_self:
self = args[0]
args = args[1:]
else:
self = cache_store
now = Date.now()
try:
_cache = getattr(self, attr_name)
except Exception, _:
_cache = {}
setattr(self, attr_name, _cache)
if Random.int(100) == 0:
# REMOVE OLD CACHE
_cache = {k: v for k, v in _cache.items() if v[0]==None or v[0] < now}
setattr(self, attr_name, _cache)
timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null))
if now > timeout:
value = func(self, *args)
_cache[args] = (now + cache_store.timeout, args, value, None)
return value
if value == None:
if exception == None:
try:
value = func(self, *args)
_cache[args] = (now + cache_store.timeout, args, value, None)
return value
except Exception, e:
e = Except.wrap(e)
_cache[args] = (now + cache_store.timeout, args, None, e)
raise e
else:
raise exception
def query(self, _query):
try:
query = Query(_query, schema=self)
for n in self.namespaces:
query = n.convert(query)
if self.typed:
query = Typed().convert(query)
for s in listwrap(query.select):
if not aggregates1_4.get(s.aggregate):
Log.error("ES can not aggregate " + s.name + " because '" + s.aggregate + "' is not a recognized aggregate")
frum = query["from"]
if isinstance(frum, Query):
result = self.query(frum)
q2 = query.copy()
q2.frum = result
return qb.run(q2)
if is_deepop(self._es, query):
return es_deepop(self._es, query)
if is_aggsop(self._es, query):
return es_aggsop(self._es, frum, query)
if is_setop(self._es, query):
return es_setop(self._es, query)
if es09_setop.is_setop(query):
return es09_setop.es_setop(self._es, None, query)
if es09_aggop.is_aggop(query):
return es09_aggop.es_aggop(self._es, None, query)
Log.error("Can not handle")
except Exception, e:
e = Except.wrap(e)
if "Data too large, data for" in e:
http.post(self._es.cluster.path+"/_cache/clear")
Log.error("Problem (Tried to clear Elasticsearch cache)", e)
Log.error("problem", e)
#NORMAL RE-RAISE
def test_python_re_raise(self):
def problem_y():
raise Exception("this is the root cause")
def problem_x():
try:
problem_y()
except Exception, f:
raise f
try:
problem_x()
except Exception, e:
e = Except.wrap(e)
self.assertEqual(e.cause, None) # REALLY, THE CAUSE IS problem_y()
def test_contains_from_zip_error(self):
def bad_unzip():
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
return decompressor.decompress(b'invlaid zip file')
try:
bad_unzip()
assert False
except Exception, e:
e = Except.wrap(e)
if "incorrect header check" in e:
pass
else:
try:
if DEBUG:
_Log.note("reading file {{path}}", path=path)
content = File(path).read()
except Exception, e:
content = None
_Log.error("Could not read file {{filename}}", filename=path, cause=e)
try:
new_value = _convert.json2value(content, params=ref.query, flexible=True, leaves=True)
except Exception, e:
if not _Except:
_late_import()
e = _Except.wrap(e)
try:
new_value = _convert.ini2value(content)
except Exception, f:
raise _Log.error("Can not read {{file}}", file=path, cause=e)
new_value = _replace_ref(new_value, ref)
return new_value
def get_http(ref, url):
from pyLibrary.env import http
params = url.query
new_value = _convert.json2value(http.get(ref), params=params, flexible=True, leaves=True)
return new_value
def int_list_packer(term, values):
"""
return singletons, ranges and exclusions
"""
DENSITY = 10 # a range can have holes, this is inverse of the hole density
MIN_RANGE = 20 # min members before a range is allowed to be used
singletons = set()
ranges = []
exclude = set()
sorted = qb.sort(values)
last = sorted[0]
curr_start = last
curr_excl = set()
for v in sorted[1::]:
if v <= last + 1:
pass
elif v - last > 3:
# big step, how do we deal with it?
if last == curr_start:
# not a range yet, so just add as singlton
singletons.add(last)
elif last - curr_start - len(curr_excl) < MIN_RANGE or (
(last - curr_start) < len(curr_excl) * DENSITY):
# small ranges are singletons, sparse ranges are singletons
singletons |= set(range(curr_start, last + 1))
singletons -= curr_excl
else:
# big enough, and dense enough range
ranges.append({"gte": curr_start, "lte": last})
exclude |= curr_excl
curr_start = v
curr_excl = set()
else:
if 1 + last - curr_start >= len(curr_excl) * DENSITY:
# high density, keep track of excluded and continue
add_me = set(range(last + 1, v))
curr_excl |= add_me
elif 1 + last - curr_start - len(curr_excl) < MIN_RANGE:
# not big enough, convert range to singletons
new_singles = set(range(curr_start, last + 1)) - curr_excl
singletons = singletons | new_singles
curr_start = v
curr_excl = set()
else:
ranges.append({"gte": curr_start, "lte": last})
exclude |= curr_excl
curr_start = v
curr_excl = set()
last = v
if last == curr_start:
# not a range yet, so just add as singlton
singletons.add(last)
elif last - curr_start - len(curr_excl) < MIN_RANGE or (
(last - curr_start) < len(curr_excl) * DENSITY):
# small ranges are singletons, sparse ranges are singletons
singletons |= set(range(curr_start, last + 1))
singletons -= curr_excl
else:
# big enough, and dense enough range
ranges.append({"gte": curr_start, "lte": last})
exclude |= curr_excl
if ranges:
r = {"or": [{"range": {term: r}} for r in ranges]}
if exclude:
r = {"and": [r, {"not": {"terms": {term: qb.sort(exclude)}}}]}
if singletons:
return {"or": [{"terms": {term: qb.sort(singletons)}}, r]}
else:
return r
else:
raise Except("no packing possible")
_to_ascii_dict(kwargs[b"headers"])
else:
_to_ascii_dict(kwargs.get(b"headers"))
except Exception, e:
Log.error("Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Thread.sleep(retry.sleep)
try:
return session.request(method=method, url=url, **kwargs)
except Exception, e:
errors.append(Except.wrap(e))
if " Read timed out." in errors[0]:
Log.error("Tried {{times}} times: Timeout failure (timeout was {{timeout}}", timeout=timeout, times=retry.times, cause=errors[0])
else:
Log.error("Tried {{times}} times: Request failure of {{url}}", url=url, times=retry.times, cause=errors[0])
def _to_ascii_dict(headers):
if headers is None:
return
for k, v in copy(headers).items():
if isinstance(k, unicode):
del headers[k]
if isinstance(v, unicode):
headers[k.encode("ascii")] = v.encode("ascii")
