def solve(): try: data = convert.json2value(convert.utf82unicode(flask.request.data)) solved = noop.solve(data) response_data = convert.unicode2utf8(convert.value2json(solved)) return Response( response_data, direct_passthrough=True, # FOR STREAMING status=200, headers={ "access-control-allow-origin": "*", "content-type": "application/json" } ) except Exception, e: e = Except.wrap(e) Log.warning("Could not process", cause=e) e = e.as_dict() return Response( convert.unicode2utf8(convert.value2json(e)), status=400, headers={ "access-control-allow-origin": "*", "content-type": "application/json" } )
def json2value(json_string, params={}, flexible=False, leaves=False): """ :param json_string: THE JSON :param params: STANDARD JSON PARAMS :param flexible: REMOVE COMMENTS :param leaves: ASSUME JSON KEYS ARE DOT-DELIMITED :return: Python value """ if isinstance(json_string, str): Log.error("only unicode json accepted") try: if flexible: # REMOVE """COMMENTS""", # COMMENTS, //COMMENTS, AND \n \r # DERIVED FROM https://github.com/jeads/datasource/blob/master/datasource/bases/BaseHub.py# L58 json_string = re.sub(r"\"\"\".*?\"\"\"", r"\n", json_string, flags=re.MULTILINE) json_string = "\n".join(remove_line_comment(l) for l in json_string.split("\n")) # ALLOW DICTIONARY'S NAME:VALUE LIST TO END WITH COMMA json_string = re.sub(r",\s*\}", r"}", json_string) # ALLOW LISTS TO END WITH COMMA json_string = re.sub(r",\s*\]", r"]", json_string) if params: json_string = expand_template(json_string, params) # LOOKUP REFERENCES value = wrap(json_decoder(json_string)) if leaves: value = wrap_leaves(value) return value except Exception, e: e = Except.wrap(e) if "Expecting '" in e and "' delimiter: line" in e: line_index = int(strings.between(e.message, " line ", " column ")) - 1 column = int(strings.between(e.message, " column ", " ")) - 1 line = json_string.split("\n")[line_index].replace("\t", " ") if column > 20: sample = "..." + line[column - 20:] pointer = " " + (" " * 20) + "^" else: sample = line pointer = (" " * column) + "^" if len(sample) > 43: sample = sample[:43] + "..." Log.error("Can not decode JSON at:\n\t" + sample + "\n\t" + pointer + "\n") base_str = unicode2utf8(strings.limit(json_string, 1000)) hexx_str = bytes2hex(base_str, " ") try: char_str = " " + (" ".join(c.decode("latin1") if ord(c) >= 32 else ".") for c in base_str) except Exception: char_str = " " Log.error("Can not decode JSON:\n" + char_str + "\n" + hexx_str + "\n", e)
def write(self, template, params): try: self.queue.add({"template": template, "params": params}) return self except Exception, e: e = _Except.wrap(e) sys.stdout.write("IF YOU SEE THIS, IT IS LIKELY YOU FORGOT TO RUN Log.start() FIRST\n") raise e # OH NO!
def _kill(self): try: self.service.kill() except Exception, e: ee = Except.wrap(e) if "The operation completed successfully" in ee: return Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
def try_till_response(self, *args, **kwargs): while True: try: response = self.server.get(*args, **kwargs) return response except Exception, e: e = Except.wrap(e) if "No connection could be made because the target machine actively refused it" in e: Log.alert("Problem connecting") else: Log.error("Server raised exception", e)
def test_trace_of_simple_raises(self): try: problem_a() except Exception, e: f = Except.wrap(e) self.assertEqual(f.template, "expected exception") for i, m in enumerate(listwrap(f.trace).method): if m == "test_trace_of_simple_raises": self.assertEqual(i, 2) break else: self.fail("expecting stack to show this method")
def encode(self, value, pretty=False): if pretty: return pretty_json(value) try: scrubbed = scrub(value) return unicode(self.encoder.encode(scrubbed)) except Exception, e: from pyLibrary.debugs.logs import Log, Except e = Except.wrap(e) Log.warning("problem serializing {{type}}", type=_repr(value), cause=e) raise e
def test_full_trace_on_wrap(self): try: problem_b() except Exception, e: cause = Except.wrap(e) self.assertEqual(cause.template, "expected exception") for i, m in enumerate(listwrap(cause.trace).method): if m == "test_full_trace_on_wrap": self.assertEqual(i, 1) break else: self.fail("expecting stack to show this method")
def test_contains_from_zip_error(self): def bad_unzip(): decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) return decompressor.decompress(b'invlaid zip file') try: bad_unzip() assert False except Exception, e: e = Except.wrap(e) if "incorrect header check" in e: pass else: assert False
def _got_result(self, data, message): data = wrap(data) data._meta.count = self.count self.count += 1 if self.settings.debug: Log.note("{{data}}", data=data) if self.target_queue != None: try: self.target_queue.add(data) message.ack() except Exception, e: e = Except.wrap(e) if not self.target_queue.closed: # EXPECTED TO HAPPEN, THIS THREAD MAY HAVE BEEN AWAY FOR A WHILE raise e
def _got_result(self, data, message): data = wrap(data) data._meta.count = self.count self.count += 1 if self.settings.debug: Log.note("{{data}}", data= data) if self.target_queue != None: try: self.target_queue.add(data) message.ack() except Exception, e: e = Except.wrap(e) if not self.target_queue.closed: # EXPECTED TO HAPPEN, THIS THREAD MAY HAVE BEEN AWAY FOR A WHILE raise e
def value2json(obj, pretty=False): try: json = json_encoder(obj, pretty=pretty) if json == None: Log.note(str(type(obj)) + " is not valid{{type}}JSON", type= " (pretty) " if pretty else " ") Log.error("Not valid JSON: " + str(obj) + " of type " + str(type(obj))) return json except Exception, e: e = Except.wrap(e) try: json = pypy_json_encode(obj) return json except Exception: pass Log.error("Can not encode into JSON: {{value}}", value=repr(obj), cause=e)
def utf82unicode(value): """ WITH EXPLANATION FOR FAILURE """ try: return value.decode("utf8") except Exception, e: if not _Log: _late_import() if not isinstance(value, basestring): _Log.error("Can not _convert {{type}} to unicode because it's not a string", type= type(value).__name__) e = _Except.wrap(e) for i, c in enumerate(value): try: c.decode("utf8") except Exception, f: _Log.error("Can not _convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])
def query(self, _query): try: if not self.ready: Log.error("Must use with clause for any instance of FromES") query = Query(_query, schema=self) # try: # frum = self.get_columns(query["from"]) # mvel = _MVEL(frum) # except Exception, e: # mvel = None # Log.warning("TODO: Fix this", e) # for s in listwrap(query.select): if not aggregates1_4[s.aggregate]: Log.error("ES can not aggregate " + self.select[0].name + " because '" + self.select[0].aggregate + "' is not a recognized aggregate") frum = query["from"] if isinstance(frum, Query): result = self.query(frum) q2 = query.copy() q2.frum = result return qb.run(q2) if is_aggsop(self._es, query): return es_aggsop(self._es, frum, query) if is_fieldop(self._es, query): return es_fieldop(self._es, query) if is_setop(self._es, query): return es_setop(self._es, query) Log.error("Can not handle") except Exception, e: e = Except.wrap(e) if "Data too large, data for" in e: http.post(self._es.cluster.path + "/_cache/clear") Log.error("Problem (Tried to clear Elasticsearch cache)", e) Log.error("problem", e)
def request(method, url, zip=None, retry=None, **kwargs): """ JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES DEMANDS data IS ONE OF: * A JSON-SERIALIZABLE STRUCTURE, OR * LIST OF JSON-SERIALIZABLE STRUCTURES, OR * None Parameters * zip - ZIP THE REQUEST BODY, IF BIG ENOUGH * json - JSON-SERIALIZABLE STRUCTURE * retry - {"times": x, "sleep": y} STRUCTURE THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT** IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH INCLUDES url AND headers """ global _warning_sent if not default_headers and not _warning_sent: _warning_sent = True Log.warning( "The pyLibrary.env.http module was meant to add extra " "default headers to all requests, specifically the 'Referer' " "header with a URL to the project. Use the `pyLibrary.debug.constants.set()` " "function to set `pyLibrary.env.http.default_headers`" ) if isinstance(url, list): # TRY MANY URLS failures = [] for remaining, u in qb.countdown(url): try: response = request(method, u, zip=zip, retry=retry, **kwargs) if Math.round(response.status_code, decimal=-2) not in [400, 500]: return response if not remaining: return response except Exception, e: e = Except.wrap(e) failures.append(e) Log.error("Tried {{num}} urls", num=len(url), cause=failures)
def output(*args): with cache_store.locker: if using_self: self = args[0] args = args[1:] else: self = cache_store now = Date.now() try: _cache = getattr(self, attr_name) except Exception, _: _cache = {} setattr(self, attr_name, _cache) if Random.int(100) == 0: # REMOVE OLD CACHE _cache = {k: v for k, v in _cache.items() if v[0]==None or v[0] < now} setattr(self, attr_name, _cache) timeout, key, value, exception = _cache.get(args, (Null, Null, Null, Null)) if now > timeout: value = func(self, *args) _cache[args] = (now + cache_store.timeout, args, value, None) return value if value == None: if exception == None: try: value = func(self, *args) _cache[args] = (now + cache_store.timeout, args, value, None) return value except Exception, e: e = Except.wrap(e) _cache[args] = (now + cache_store.timeout, args, None, e) raise e else: raise exception
def query(self, _query): try: query = Query(_query, schema=self) for n in self.namespaces: query = n.convert(query) if self.typed: query = Typed().convert(query) for s in listwrap(query.select): if not aggregates1_4.get(s.aggregate): Log.error("ES can not aggregate " + s.name + " because '" + s.aggregate + "' is not a recognized aggregate") frum = query["from"] if isinstance(frum, Query): result = self.query(frum) q2 = query.copy() q2.frum = result return qb.run(q2) if is_deepop(self._es, query): return es_deepop(self._es, query) if is_aggsop(self._es, query): return es_aggsop(self._es, frum, query) if is_setop(self._es, query): return es_setop(self._es, query) if es09_setop.is_setop(query): return es09_setop.es_setop(self._es, None, query) if es09_aggop.is_aggop(query): return es09_aggop.es_aggop(self._es, None, query) Log.error("Can not handle") except Exception, e: e = Except.wrap(e) if "Data too large, data for" in e: http.post(self._es.cluster.path+"/_cache/clear") Log.error("Problem (Tried to clear Elasticsearch cache)", e) Log.error("problem", e)
#NORMAL RE-RAISE def test_python_re_raise(self): def problem_y(): raise Exception("this is the root cause") def problem_x(): try: problem_y() except Exception, f: raise f try: problem_x() except Exception, e: e = Except.wrap(e) self.assertEqual(e.cause, None) # REALLY, THE CAUSE IS problem_y() def test_contains_from_zip_error(self): def bad_unzip(): decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) return decompressor.decompress(b'invlaid zip file') try: bad_unzip() assert False except Exception, e: e = Except.wrap(e) if "incorrect header check" in e: pass else:
try: if DEBUG: _Log.note("reading file {{path}}", path=path) content = File(path).read() except Exception, e: content = None _Log.error("Could not read file {{filename}}", filename=path, cause=e) try: new_value = _convert.json2value(content, params=ref.query, flexible=True, leaves=True) except Exception, e: if not _Except: _late_import() e = _Except.wrap(e) try: new_value = _convert.ini2value(content) except Exception, f: raise _Log.error("Can not read {{file}}", file=path, cause=e) new_value = _replace_ref(new_value, ref) return new_value def get_http(ref, url): from pyLibrary.env import http params = url.query new_value = _convert.json2value(http.get(ref), params=params, flexible=True, leaves=True) return new_value
_to_ascii_dict(kwargs[b"headers"]) else: _to_ascii_dict(kwargs.get(b"headers")) except Exception, e: Log.error("Request setup failure on {{url}}", url=url, cause=e) errors = [] for r in range(retry.times): if r: Thread.sleep(retry.sleep) try: return session.request(method=method, url=url, **kwargs) except Exception, e: errors.append(Except.wrap(e)) if " Read timed out." in errors[0]: Log.error("Tried {{times}} times: Timeout failure (timeout was {{timeout}}", timeout=timeout, times=retry.times, cause=errors[0]) else: Log.error("Tried {{times}} times: Request failure of {{url}}", url=url, times=retry.times, cause=errors[0]) def _to_ascii_dict(headers): if headers is None: return for k, v in copy(headers).items(): if isinstance(k, unicode): del headers[k] if isinstance(v, unicode): headers[k.encode("ascii")] = v.encode("ascii")