def float2json(value): """ CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY :param value: float, int, long, Decimal :return: unicode """ if value == 0: return u'0' try: sign = "-" if value < 0 else "" value = abs(value) sci = value.__format__(".15e") mantissa, str_exp = sci.split("e") digits, more_digits = _snap_to_base_10(mantissa) int_exp = int(str_exp) + more_digits if int_exp > 15: return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp) elif int_exp >= 0: return sign + (digits[:1 + int_exp] + '.' + digits[1 + int_exp:].rstrip('0')).rstrip('.') elif -4 < int_exp: digits = ("0" * (-int_exp)) + digits return sign + (digits[:1] + '.' + digits[1:].rstrip('0')).rstrip('.') else: return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp) except Exception as e: from mo_logs import Log Log.error("not expected", e)
def wrap(cls, e, stack_depth=0): """ ENSURE THE STACKTRACE AND CAUSAL CHAIN IS CAPTURED, PLUS ADD FEATURES OF Except :param e: AN EXCEPTION OF ANY TYPE :param stack_depth: HOW MANY CALLS TO TAKE OFF THE TOP OF THE STACK TRACE :return: A Except OBJECT OF THE SAME """ if e == None: return Null elif isinstance(e, (list, Except)): return e elif isinstance(e, Mapping): e.cause = unwraplist([Except.wrap(c) for c in listwrap(e.cause)]) return Except(**e) else: tb = getattr(e, '__traceback__', None) if tb is not None: trace = _parse_traceback(tb) else: trace = _extract_traceback(0) cause = Except.wrap(getattr(e, '__cause__', None)) if hasattr(e, "message") and e.message: output = Except(type=ERROR, template=text_type(e.message), trace=trace, cause=cause) else: output = Except(type=ERROR, template=text_type(e), trace=trace, cause=cause) trace = extract_stack(stack_depth + 2) # +2 = to remove the caller, and it's call to this' Except.wrap() output.trace.extend(trace) return output
def _annotate(cls, item, timestamp, stack_depth): """ :param itemt: A LogItemTHE TYPE OF MESSAGE :param stack_depth: FOR TRACKING WHAT LINE THIS CAME FROM :return: """ item.timestamp = timestamp item.machine = machine_metadata item.template = strings.limit(item.template, 10000) item.format = strings.limit(item.format, 10000) if item.format == None: format = text_type(item) else: format = item.format.replace("{{", "{{params.") if not format.startswith("\n") and format.find("\n") > -1: format = "\n" + format if cls.trace: log_format = item.format = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" - ({{location.method}}) - " + format f = sys._getframe(stack_depth + 1) item.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename), "method": text_type(f.f_code.co_name) } thread = _Thread.current() item.thread = {"name": thread.name, "id": thread.id} else: log_format = item.format = "{{timestamp|datetime}} - " + format cls.main_log.write(log_format, item.__data__())
def quote_value(self, value): """ convert values to mysql code for the same mostly delegate directly to the mysql lib, but some exceptions exist """ try: if value == None: return SQL_NULL elif isinstance(value, SQL): return self.quote_sql(value.template, value.param) elif isinstance(value, text_type): return SQL("'" + value.replace("'", "''") + "'") elif isinstance(value, Mapping): return self.quote_value(json_encode(value)) elif Math.is_number(value): return SQL(text_type(value)) elif isinstance(value, datetime): return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif isinstance(value, Date): return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif hasattr(value, '__iter__'): return self.quote_value(json_encode(value)) else: return self.quote_value(text_type(value)) except Exception as e: Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)
def quote_value(value): """ convert values to mysql code for the same mostly delegate directly to the mysql lib, but some exceptions exist """ try: if value == None: return SQL_NULL elif isinstance(value, SQL): return quote_sql(value.template, value.param) elif is_text(value): return SQL("'" + "".join(ESCAPE_DCT.get(c, c) for c in value) + "'") elif is_data(value): return quote_value(json_encode(value)) elif is_number(value): return SQL(text_type(value)) elif isinstance(value, datetime): return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif isinstance(value, Date): return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif hasattr(value, '__iter__'): return quote_value(json_encode(value)) else: return quote_value(text_type(value)) except Exception as e: Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)
def wrap(cls, e, stack_depth=0): """ ENSURE THE STACKTRACE AND CAUSAL CHAIN IS CAPTURED, PLUS ADD FEATURES OF Except :param e: AN EXCEPTION OF ANY TYPE :param stack_depth: HOW MANY CALLS TO TAKE OFF THE TOP OF THE STACK TRACE :return: A Except OBJECT OF THE SAME """ if e == None: return Null elif isinstance(e, (list, Except)): return e elif is_data(e): e.cause = unwraplist([Except.wrap(c) for c in listwrap(e.cause)]) return Except(**e) else: tb = getattr(e, '__traceback__', None) if tb is not None: trace = _parse_traceback(tb) else: trace = _extract_traceback(0) cause = Except.wrap(getattr(e, '__cause__', None)) if hasattr(e, "message") and e.message: output = Except(context=ERROR, template=text_type(e.message), trace=trace, cause=cause) else: output = Except(context=ERROR, template=text_type(e), trace=trace, cause=cause) trace = extract_stack(stack_depth + 2) # +2 = to remove the caller, and it's call to this' Except.wrap() output.trace.extend(trace) return output
def utf82unicode(value): """ WITH EXPLANATION FOR FAILURE """ try: return value.decode("utf8") except Exception as e: if not _Log: _late_import() if not is_binary(value): _Log.error("Can not convert {{type}} to unicode because it's not bytes", type= type(value).__name__) e = _Except.wrap(e) for i, c in enumerate(value): try: c.decode("utf8") except Exception as f: _Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)]) try: latin1 = text_type(value.decode("latin1")) _Log.error("Can not explain conversion failure, but seems to be latin1", e) except Exception: pass try: a = text_type(value.decode("latin1")) _Log.error("Can not explain conversion failure, but seems to be latin1", e) except Exception: pass _Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)
def _make_range_domain(self, domain, column_name): width = (domain.max - domain.min) / domain.interval digits = mo_math.floor(mo_math.log10(width - 1)) if digits == 0: value = "a.value" else: value = SQL("+").join("1" + ("0" * j) + "*" + text_type(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1)) if domain.interval == 1: if domain.min == 0: domain = (SQL_SELECT + value + column_name + SQL_FROM + "__digits__ a") else: domain = (SQL_SELECT + sql_iso(value) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a") else: if domain.min == 0: domain = (SQL_SELECT + value + " * " + quote_value(domain.interval) + column_name + SQL_FROM + "__digits__ a") else: domain = ( SQL_SELECT + sql_iso(value + " * " + quote_value(domain.interval)) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a") for j in range(digits): domain += SQL_INNER_JOIN + "__digits__" + text_type( chr(ord(b'a') + j + 1)) + " ON " + SQL_TRUE domain += SQL_WHERE + value + " < " + quote_value(width) return domain
def note(template, **params): if not is_text(template): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] log_params = wrap( { "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata, "context": exceptions.NOTE, } ) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if Log.trace: log_template = ( '{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - "{{location.file}}:{{location.line}}" ({{location.method}}) - ' + template.replace("{{", "{{params.") ) f = sys._getframe(1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name), } else: log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.") Log.main_log.write(log_template, log_params)
def note(template, **params): if not isinstance(template, text_type): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] log_params = wrap({ "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata, "context": exceptions.NOTE }) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if Log.trace: log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace( "{{", "{{params.") f = sys._getframe(1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name) } else: log_template = "{{timestamp|datetime}} - " + template.replace( "{{", "{{params.") Log.main_log.write(log_template, log_params)
def to_python(self, not_null=False, boolean=False, many=False): return ( "row[" + text_type(self.var) + "] if 0<=" + text_type(self.var) + "<len(row) else None" )
def sort_table(result): """ SORT ROWS IN TABLE, EVEN IF ELEMENTS ARE JSON """ data = wrap([{text_type(i): v for i, v in enumerate(row)} for row in result.data]) sort_columns = jx.sort(set(jx.get_columns(data, leaves=True).name)) data = jx.sort(data, sort_columns) result.data = [tuple(row[text_type(i)] for i in range(len(result.header))) for row in data]
def tab(value): if isinstance(value, Mapping): h, d = zip(*wrap(value).leaves()) return \ "\t".join(map(value2json, h)) + \ "\n" + \ "\t".join(map(value2json, d)) else: text_type(value)
def tab(value): """ convert single value to tab-delimited form, including a header :param value: :return: """ if isinstance(value, Mapping): h, d = zip(*wrap(value).leaves()) return ("\t".join(map(value2json, h)) + "\n" + "\t".join(map(value2json, d))) else: text_type(value)
def tab(value): """ convert single value to tab-delimited form, including a header :param value: :return: """ if is_data(value): h, d = transpose(*wrap(value).leaves()) return ("\t".join(map(value2json, h)) + CR + "\t".join(map(value2json, d))) else: text_type(value)
def box(script): """ :param es_script: :return: TEXT EXPRESSION WITH NON OBJECTS BOXED """ if script.type is BOOLEAN: return "Boolean.valueOf(" + text_type(script.expr) + ")" elif script.type is INTEGER: return "Integer.valueOf(" + text_type(script.expr) + ")" elif script.type is NUMBER: return "Double.valueOf(" + text_type(script.expr) + ")" else: return script.expr
def _convert(v): if v is None: return null_script if v is True: return true_script if v is False: return false_script class_ = v.__class__ if class_ is text_type: return EsScript(type=STRING, expr=quote(v), frum=self, schema=schema) if class_ in integer_types: if MIN_INT32 <= v <= MAX_INT32: return EsScript(type=INTEGER, expr=text_type(v), frum=self, schema=schema) else: return EsScript(type=INTEGER, expr=text_type(v) + "L", frum=self, schema=schema) if class_ is float: return EsScript(type=NUMBER, expr=text_type(v) + "D", frum=self, schema=schema) if class_ in data_types: return EsScript( type=OBJECT, expr="[" + ", ".join( quote(k) + ": " + _convert(vv) for k, vv in v.items()) + "]", frum=self, schema=schema, ) if class_ in (FlatList, list, tuple): return EsScript( type=OBJECT, expr="[" + ", ".join(_convert(vv).expr for vv in v) + "]", frum=self, schema=schema, ) if class_ is Date: return EsScript(type=NUMBER, expr=text_type(v.unix), frum=self, schema=schema)
def _convert(v): if v is None: return NULL.to_es_script(schema) if v is True: return EsScript( type=BOOLEAN, expr="true", frum=self ) if v is False: return EsScript( type=BOOLEAN, expr="false", frum=self ) if isinstance(v, text_type): return EsScript( type=STRING, expr=quote(v), frum=self ) if isinstance(v, int): return EsScript( type=INTEGER, expr=text_type(v), frum=self ) if isinstance(v, float): return EsScript( type=NUMBER, expr=text_type(v), frum=self ) if isinstance(v, dict): return EsScript( type=OBJECT, expr="[" + ", ".join(quote(k) + ": " + _convert(vv) for k, vv in v.items()) + "]", frum=self ) if isinstance(v, (list, tuple)): return EsScript( type=OBJECT, expr="[" + ", ".join(_convert(vv).expr for vv in v) + "]", frum=self ) if isinstance(v, Date): return EsScript( type=NUMBER, expr=text_type(v.unix), frum=self )
def table2csv(table_data): """ :param table_data: expecting a list of tuples :return: text in nice formatted csv """ text_data = [tuple(value2json(vals, pretty=True) for vals in rows) for rows in table_data] col_widths = [max(len(text) for text in cols) for cols in zip(*text_data)] template = ", ".join( "{{" + text_type(i) + "|left_align(" + text_type(w) + ")}}" for i, w in enumerate(col_widths) ) text = "\n".join(expand_template(template, d) for d in text_data) return text
def encode(self, value, pretty=False): if pretty: return pretty_json(value) try: scrubbed = scrub(value) return text_type(self.encoder(scrubbed)) except Exception as e: from mo_logs.exceptions import Except from mo_logs import Log e = Except.wrap(e) Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e) raise e
def note(cls, template, default_params={}, stack_depth=0, log_context=None, **more_params): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if not isinstance(template, text_type): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = set_default( { "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata }, log_context, {"context": exceptions.NOTE}) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace( "{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name) } thread = _Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace( "{{", "{{params.") cls.main_log.write(log_template, log_params)
def tab(value): """ convert single value to tab-delimited form, including a header :param value: :return: """ if is_data(value): h, d = transpose(*wrap(value).leaves()) return ( "\t".join(map(value2json, h)) + CR + "\t".join(map(value2json, d)) ) else: text_type(value)
def note( cls, template, default_params={}, stack_depth=0, log_context=None, **more_params ): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if not isinstance(template, text_type): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = set_default({ "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata }, log_context, {"context": exceptions.NOTE}) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace("{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name) } thread = _Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.") cls.main_log.write(log_template, log_params)
def __init__(self, ident): self.id = ident if ident != -1: self.name = "Unknown Thread " + text_type(ident) self.child_lock = allocate_lock() self.children = [] self.cprofiler = None
def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None): self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD) self.rate = coalesce(rate, HG_REQUEST_PER_SECOND) self.cache_locker = Lock() self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR self.no_cache = {} # VERY SHORT TERM CACHE self.workers = [] self.todo = Queue(APP_NAME+" todo") self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds)) self.url = URL(source.url) self.db = Sqlite(database) self.inbound_rate = RateLogger("Inbound") self.outbound_rate = RateLogger("hg.mo") if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data: with self.db.transaction() as t: t.execute( "CREATE TABLE cache (" " path TEXT PRIMARY KEY, " " headers TEXT, " " response TEXT, " " timestamp REAL " ")" ) self.threads = [ Thread.run(APP_NAME+" worker" + text_type(i), self._worker) for i in range(CONCURRENCY) ] self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter) self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def __getitem__(self, key): if key == None: return Null if key == ".": output = self._internal_dict if isinstance(output, Mapping): return self else: return output key = text_type(key) d = self._internal_dict if key.find(".") >= 0: seq = _split_field(key) for n in seq: if isinstance(d, NullType): d = NullType(d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL elif isinstance(d, list): d = [_getdefault(dd, n) for dd in d] else: d = _getdefault(d, n) # EVERYTHING ELSE TREATS n AS LITERAL return wrap(d) else: o = d.get(key) if o == None: return NullType(d, key) return wrap(o)
def float2json(value): """ CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY :param value: float, int, long, Decimal :return: unicode """ if value == 0: return u'0' try: sign = "-" if value < 0 else "" value = abs(value) sci = value.__format__(".15e") mantissa, exp = sci.split("e") exp = int(exp) if 0 <= exp: digits = u"".join(mantissa.split(".")) return sign + (digits[:1 + exp] + u"." + digits[1 + exp:].rstrip('0')).rstrip(".") elif -4 < exp: digits = ("0" * (-exp)) + u"".join(mantissa.split(".")) return sign + (digits[:1] + u"." + digits[1:].rstrip('0')).rstrip(".") else: return sign + mantissa.rstrip("0") + u"e" + text_type(exp) except Exception as e: from mo_logs import Log Log.error("not expected", e)
def value2json(obj, pretty=False, sort_keys=False, keep_whitespace=True): """ :param obj: THE VALUE TO TURN INTO JSON :param pretty: True TO MAKE A MULTI-LINE PRETTY VERSION :param sort_keys: True TO SORT KEYS :param keep_whitespace: False TO strip() THE WHITESPACE IN THE VALUES :return: """ if FIND_LOOPS: obj = scrub(obj, scrub_text=_keep_whitespace if keep_whitespace else _trim_whitespace()) try: json = json_encoder(obj, pretty=pretty) if json == None: Log.note(str(type(obj)) + " is not valid{{type}}JSON", type=" (pretty) " if pretty else " ") Log.error("Not valid JSON: " + str(obj) + " of type " + str(type(obj))) return json except Exception as e: e = Except.wrap(e) try: json = pypy_json_encode(obj) return json except Exception: pass Log.error("Can not encode into JSON: {{value}}", value=text_type(repr(obj)), cause=e)
def __getitem__(self, key): if key == None: return Null if key == ".": output = self._internal_dict if _get(output, CLASS) in data_types: return self else: return output key = text_type(key) d = self._internal_dict if key.find(".") >= 0: seq = _split_field(key) for n in seq: if _get(d, CLASS) is NullType: d = NullType( d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL elif is_list(d): d = [_getdefault(dd, n) for dd in d] else: d = _getdefault(d, n) # EVERYTHING ELSE TREATS n AS LITERAL return wrap(d) else: o = d.get(key) if o == None: return NullType(d, key) return wrap(o)
def encode(self, value, pretty=False): if pretty: return pretty_json(value) try: with Timer("scrub", too_long=0.1): scrubbed = scrub(value) with Timer("encode", too_long=0.1): return text_type(self.encoder(scrubbed)) except Exception as e: from mo_logs.exceptions import Except from mo_logs import Log e = Except.wrap(e) Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e) raise e
def main(): try: settings = startup.read_settings() with startup.SingleInstance(settings.args.filename): constants.set(settings.constants) Log.start(settings.debug) extractor = Extract(settings) def extract(please_stop): with MySQL(**settings.snowflake.database) as db: with db.transaction(): for kwargs in extractor.queue: if please_stop: break try: extractor.extract(db=db, please_stop=please_stop, **kwargs) except Exception as e: Log.warning("Could not extract", cause=e) extractor.queue.add(kwargs) for i in range(settings.extract.threads): Thread.run("extract #" + text_type(i), extract) please_stop = Signal() Thread.wait_for_shutdown_signal(please_stop=please_stop, allow_exit=True, wait_forever=False) except Exception as e: Log.warning("Problem with data extraction", e) finally: Log.stop()
def test_single_file_list(config, app): url = "http://localhost:" + text_type(config.flask.port) + "/tuid" response = http.post_json( url, json={ "meta": { "format": "list" }, "from": "files", "where": { "and": [{ "eq": { "revision": "29dcc9cb77c372c97681a47496488ec6c623915d" } }, { "in": { "path": ["gfx/thebes/gfxFontVariations.h"] } }, { "eq": { "branch": "mozilla-central" } }] } }) list_response = response.data tuids = list_response[0].tuids assert len(tuids) == 41 # 41 lines expected assert len(set(tuids)) == 41 # tuids much be unique
def get(url): """ USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON """ url = text_type(url) if url.find("://") == -1: Log.error("{{url}} must have a prototcol (eg http://) declared", url=url) base = URL("") if url.startswith("file://") and url[7] != "/": if os.sep == "\\": base = URL("file:///" + os.getcwd().replace(os.sep, "/").rstrip("/") + "/.") else: base = URL("file://" + os.getcwd().rstrip("/") + "/.") elif url[url.find("://") + 3] != "/": Log.error("{{url}} must be absolute", url=url) phase1 = _replace_ref(wrap({"$ref": url}), base) # BLANK URL ONLY WORKS IF url IS ABSOLUTE try: phase2 = _replace_locals(phase1, [phase1]) return wrap(phase2) except Exception as e: Log.error("problem replacing locals in\n{{phase1}}", phase1=phase1, cause=e)
def format(self, format="%Y-%m-%d %H:%M:%S"): try: return text_type(unix2datetime(self.unix).strftime(format)) except Exception as e: from mo_logs import Log Log.error("Can not format {{value}} with {{format}}", value=unix2datetime(self.unix), format=format, cause=e)
def _find_revision(self, revision): please_stop = False locker = Lock() output = [] queue = Queue("branches", max=2000) queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"]) queue.add(THREAD_STOP) problems = [] def _find(please_stop): for b in queue: if please_stop: return try: url = b.url + "json-info?node=" + revision rev = self.get_revision(Revision(branch=b, changeset={"id": revision})) with locker: output.append(rev) Log.note("Revision found at {{url}}", url=url) except Exception as f: problems.append(f) threads = [] for i in range(3): threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop)) for t in threads: with assert_no_exception: t.join() return output
def queue_consumer(pull_queue, please_stop=None): queue = aws.Queue(pull_queue) time_offset = None request_count = 0 while not please_stop: request = queue.pop(till=please_stop) if please_stop: break if not request: Log.note("Nothing in queue, pausing for 5 seconds...") (please_stop | Till(seconds=5)).wait() continue if SKIP_TRY_REQUESTS and 'try' in request.where['and'].eq.branch: Log.note("Skipping try revision.") queue.commit() continue now = Date.now().unix if time_offset is None: time_offset = now - request.meta.request_time next_request = request.meta.request_time + time_offset if next_request > now: Log.note("Next request in {{wait_time}}", wait_time=Duration(seconds=next_request - now)) Till(till=next_request).wait() Thread.run("request "+text_type(request_count), one_request, request) request_count += 1 queue.commit()
def problem_serializing(value, e=None): """ THROW ERROR ABOUT SERIALIZING """ from mo_logs import Log try: typename = type(value).__name__ except Exception: typename = "<error getting name>" try: rep = text_type(repr(value)) except Exception as _: rep = None if rep == None: Log.error( "Problem turning value of type {{type}} to json", type=typename, cause=e ) else: Log.error( "Problem turning value ({{value}}) of type {{type}} to json", value=rep, type=typename, cause=e )
def latin12unicode(value): if isinstance(value, text_type): Log.error("can not convert unicode from latin1") try: return text_type(value.decode('latin1')) except Exception as e: Log.error("Can not convert {{value|quote}} to unicode", value=value)
def to_es_script(self, schema): return EsScript( type=NUMBER, expr=text_type(Date(self.value).unix), frum=self, schema=schema )
def problem_serializing(value, e=None): """ THROW ERROR ABOUT SERIALIZING """ from mo_logs import Log try: typename = type(value).__name__ except Exception: typename = "<error getting name>" try: rep = text_type(repr(value)) except Exception as _: rep = None if rep == None: Log.error("Problem turning value of type {{type}} to json", type=typename, cause=e) else: Log.error("Problem turning value ({{value}}) of type {{type}} to json", value=rep, type=typename, cause=e)
def queue_consumer(pull_queue, please_stop=None): queue = aws.Queue(pull_queue) time_offset = None request_count = 0 while not please_stop: request = queue.pop(till=please_stop) if please_stop: break if not request: Log.note("Nothing in queue, pausing for 5 seconds...") (please_stop | Till(seconds=5)).wait() continue if SKIP_TRY_REQUESTS and 'try' in request.where['and'].eq.branch: Log.note("Skipping try revision.") queue.commit() continue now = Date.now().unix if time_offset is None: time_offset = now - request.meta.request_time next_request = request.meta.request_time + time_offset if next_request > now: Log.note("Next request in {{wait_time}}", wait_time=Duration(seconds=next_request - now)) Till(till=next_request).wait() Thread.run("request " + text_type(request_count), one_request, request) request_count += 1 queue.commit()
def quote_value(self, value): """ convert values to mysql code for the same mostly delegate directly to the mysql lib, but some exceptions exist """ try: if value == None: return SQL_NULL elif isinstance(value, SQL): if not value.param: # value.template CAN BE MORE THAN A TEMPLATE STRING return self.quote_sql(value.template) param = {k: self.quote_sql(v) for k, v in value.param.items()} return SQL(expand_template(value.template, param)) elif isinstance(value, text_type): return SQL(self.db.literal(value)) elif isinstance(value, Mapping): return SQL(self.db.literal(json_encode(value))) elif Math.is_number(value): return SQL(text_type(value)) elif isinstance(value, datetime): return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif isinstance(value, Date): return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')") elif hasattr(value, '__iter__'): return SQL(self.db.literal(json_encode(value))) else: return self.db.literal(value) except Exception as e: Log.error("problem quoting SQL", e)
def __getitem__(self, key): if key == None: return Null if key == ".": output = _get(self, "_dict") if isinstance(output, Mapping): return self else: return output key = text_type(key) d = _get(self, "_dict") if key.find(".") >= 0: seq = _split_field(key) for n in seq: if isinstance(d, NullType): d = NullType( d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL elif isinstance(d, list): d = [_getdefault(dd, n) for dd in d] else: d = _getdefault(d, n) # EVERYTHING ELSE TREATS n AS LITERAL return wrap(d) else: o = d.get(key) if o == None: return NullType(d, key) return wrap(o)
def unexpected( cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params ): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) if cause and not isinstance(cause, Except): cause = Except(exceptions.UNEXPECTED, text_type(cause), trace=exceptions._extract_traceback(0)) trace = exceptions.extract_stack(1) e = Except(type=exceptions.UNEXPECTED, template=template, params=params, cause=cause, trace=trace) Log.note( "{{error}}", error=e, log_context=set_default({"context": exceptions.WARNING}, log_context), stack_depth=stack_depth + 1 )
def write(self, template, params): try: params.template = strings.limit(params.template, 2000) params.format = None self.queue.add({"value": _deep_json_to_string(params, 3)}, timeout=3 * 60) except Exception as e: sys.stdout.write(text_type(Except.wrap(e))) return self
def unicode_key(key): """ CONVERT PROPERTY VALUE TO QUOTED NAME OF SAME """ if not isinstance(key, (text_type, binary_type)): from mo_logs import Log Log.error("{{key|quote}} is not a valid key", key=key) return quote(text_type(key))
def to_es(self, schema, query_path="."): if self.children: return {"aggs": { name: t.to_es(schema, query_path) for i, t in enumerate(self.children) for name in [t.name if t.name else "_" + text_type(i)] }} else: return {}
def __setattr__(self, key, value): key = text_type(key) d = _get(self, "__dict__") o = wrap(d["_obj"]) k = d["__key__"] seq = [k] + [key] _assign_to_null(o, seq, value)
def unicode(value): """ Convert to a unicode string :param value: any value :return: unicode """ if value == None: return "" return text_type(value)
def index_to_coordinate(dims): """ RETURN A FUNCTION THAT WILL TAKE AN INDEX, AND MAP IT TO A coordinate IN dims :param dims: TUPLE WITH NUMBER OF POINTS IN EACH DIMENSION :return: FUNCTION """ _ = divmod # SO WE KEEP THE IMPORT num_dims = len(dims) if num_dims == 0: return _zero_dim prod = [1] * num_dims acc = 1 domain = range(0, num_dims) for i in reversed(domain): prod[i] = acc acc *= dims[i] commands = [] coords = [] for i in domain: if i == num_dims - 1: commands.append("\tc" + text_type(i) + " = index") else: commands.append("\tc" + text_type(i) + ", index = divmod(index, " + text_type(prod[i]) + ")") coords.append("c" + text_type(i)) output = None if num_dims == 1: code = ( "def output(index):\n" + "\n".join(commands) + "\n" + "\treturn " + coords[0] + "," ) else: code = ( "def output(index):\n" + "\n".join(commands) + "\n" + "\treturn " + ", ".join(coords) ) exec(code) return output
def append_query(self, query_path, es_query): domain = self.domain domain_key = domain.key value = Painless[self.edge.value] cnv = pull_functions[value.type] include = tuple(cnv(p[domain_key]) for p in domain.partitions) exists = Painless[AndOp([ InOp([value, Literal(include)]) ])].partial_eval() limit = coalesce(self.limit, len(domain.partitions)) if is_op(value, Variable): es_field = first(self.query.frum.schema.leaves(value.var)).es_column # ALREADY CHECKED THERE IS ONLY ONE match = TermsAggs( "_match", { "field": es_field, "size": limit, "order": {"_term": self.sorted} if self.sorted else None }, self ) else: match = TermsAggs( "_match", { "script": text_type(value.to_es_script(self.schema)), "size": limit }, self ) output = Aggs().add(FilterAggs("_filter", exists, None).add(match.add(es_query))) if self.edge.allowNulls: # FIND NULLS AT EACH NESTED LEVEL for p in self.schema.query_path: if p == query_path: # MISSING AT THE QUERY DEPTH output.add( NestedAggs(p).add(FilterAggs("_missing0", NotOp(exists), self).add(es_query)) ) else: # PARENT HAS NO CHILDREN, SO MISSING column = first(self.schema.values(query_path, (OBJECT, EXISTS))) output.add( NestedAggs(column.nested_path[0]).add( FilterAggs( "_missing1", NotOp(ExistsOp(Variable(column.es_column.replace(NESTED_TYPE, EXISTS_TYPE)))), self ).add(es_query) ) ) return output
def encode(self, value, pretty=False): if pretty: return pretty_json(value) try: with Timer("scrub", too_long=0.1): scrubbed = scrub(value) param = {"size": 0} with Timer("encode {{size}} characters", param=param, too_long=0.1): output = text_type(self.encoder(scrubbed)) param["size"] = len(output) return output except Exception as e: from mo_logs.exceptions import Except from mo_logs import Log e = Except.wrap(e) Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e) raise e