def float2json(value):
"""
CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY
:param value: float, int, long, Decimal
:return: unicode
"""
if value == 0:
return u'0'
try:
sign = "-" if value < 0 else ""
value = abs(value)
sci = value.__format__(".15e")
mantissa, str_exp = sci.split("e")
digits, more_digits = _snap_to_base_10(mantissa)
int_exp = int(str_exp) + more_digits
if int_exp > 15:
return sign + digits[0] + '.' + (digits[1:].rstrip('0') or
'0') + u"e" + text_type(int_exp)
elif int_exp >= 0:
return sign + (digits[:1 + int_exp] + '.' +
digits[1 + int_exp:].rstrip('0')).rstrip('.')
elif -4 < int_exp:
digits = ("0" * (-int_exp)) + digits
return sign + (digits[:1] + '.' +
digits[1:].rstrip('0')).rstrip('.')
else:
return sign + digits[0] + '.' + (digits[1:].rstrip('0') or
'0') + u"e" + text_type(int_exp)
except Exception as e:
from mo_logs import Log
Log.error("not expected", e)
def float2json(value):
"""
CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY
:param value: float, int, long, Decimal
:return: unicode
"""
if value == 0:
return u'0'
try:
sign = "-" if value < 0 else ""
value = abs(value)
sci = value.__format__(".15e")
mantissa, str_exp = sci.split("e")
digits, more_digits = _snap_to_base_10(mantissa)
int_exp = int(str_exp) + more_digits
if int_exp > 15:
return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp)
elif int_exp >= 0:
return sign + (digits[:1 + int_exp] + '.' + digits[1 + int_exp:].rstrip('0')).rstrip('.')
elif -4 < int_exp:
digits = ("0" * (-int_exp)) + digits
return sign + (digits[:1] + '.' + digits[1:].rstrip('0')).rstrip('.')
else:
return sign + digits[0] + '.' + (digits[1:].rstrip('0') or '0') + u"e" + text_type(int_exp)
except Exception as e:
from mo_logs import Log
Log.error("not expected", e)
def wrap(cls, e, stack_depth=0):
"""
ENSURE THE STACKTRACE AND CAUSAL CHAIN IS CAPTURED, PLUS ADD FEATURES OF Except
:param e: AN EXCEPTION OF ANY TYPE
:param stack_depth: HOW MANY CALLS TO TAKE OFF THE TOP OF THE STACK TRACE
:return: A Except OBJECT OF THE SAME
"""
if e == None:
return Null
elif isinstance(e, (list, Except)):
return e
elif isinstance(e, Mapping):
e.cause = unwraplist([Except.wrap(c) for c in listwrap(e.cause)])
return Except(**e)
else:
tb = getattr(e, '__traceback__', None)
if tb is not None:
trace = _parse_traceback(tb)
else:
trace = _extract_traceback(0)
cause = Except.wrap(getattr(e, '__cause__', None))
if hasattr(e, "message") and e.message:
output = Except(type=ERROR, template=text_type(e.message), trace=trace, cause=cause)
else:
output = Except(type=ERROR, template=text_type(e), trace=trace, cause=cause)
trace = extract_stack(stack_depth + 2) # +2 = to remove the caller, and it's call to this' Except.wrap()
output.trace.extend(trace)
return output
def _annotate(cls, item, timestamp, stack_depth):
"""
:param itemt: A LogItemTHE TYPE OF MESSAGE
:param stack_depth: FOR TRACKING WHAT LINE THIS CAME FROM
:return:
"""
item.timestamp = timestamp
item.machine = machine_metadata
item.template = strings.limit(item.template, 10000)
item.format = strings.limit(item.format, 10000)
if item.format == None:
format = text_type(item)
else:
format = item.format.replace("{{", "{{params.")
if not format.startswith("\n") and format.find("\n") > -1:
format = "\n" + format
if cls.trace:
log_format = item.format = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" - ({{location.method}}) - " + format
f = sys._getframe(stack_depth + 1)
item.location = {
"line": f.f_lineno,
"file": text_type(f.f_code.co_filename),
"method": text_type(f.f_code.co_name)
}
thread = _Thread.current()
item.thread = {"name": thread.name, "id": thread.id}
else:
log_format = item.format = "{{timestamp|datetime}} - " + format
cls.main_log.write(log_format, item.__data__())
def quote_value(self, value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return SQL_NULL
elif isinstance(value, SQL):
return self.quote_sql(value.template, value.param)
elif isinstance(value, text_type):
return SQL("'" + value.replace("'", "''") + "'")
elif isinstance(value, Mapping):
return self.quote_value(json_encode(value))
elif Math.is_number(value):
return SQL(text_type(value))
elif isinstance(value, datetime):
return SQL("str_to_date('" +
value.strftime("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif isinstance(value, Date):
return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif hasattr(value, '__iter__'):
return self.quote_value(json_encode(value))
else:
return self.quote_value(text_type(value))
except Exception as e:
Log.error("problem quoting SQL {{value}}",
value=repr(value),
cause=e)
def quote_value(value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return SQL_NULL
elif isinstance(value, SQL):
return quote_sql(value.template, value.param)
elif is_text(value):
return SQL("'" + "".join(ESCAPE_DCT.get(c, c) for c in value) + "'")
elif is_data(value):
return quote_value(json_encode(value))
elif is_number(value):
return SQL(text_type(value))
elif isinstance(value, datetime):
return SQL("str_to_date('" + value.strftime("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
elif isinstance(value, Date):
return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") + "', '%Y%m%d%H%i%s.%f')")
elif hasattr(value, '__iter__'):
return quote_value(json_encode(value))
else:
return quote_value(text_type(value))
except Exception as e:
Log.error("problem quoting SQL {{value}}", value=repr(value), cause=e)
def wrap(cls, e, stack_depth=0):
"""
ENSURE THE STACKTRACE AND CAUSAL CHAIN IS CAPTURED, PLUS ADD FEATURES OF Except
:param e: AN EXCEPTION OF ANY TYPE
:param stack_depth: HOW MANY CALLS TO TAKE OFF THE TOP OF THE STACK TRACE
:return: A Except OBJECT OF THE SAME
"""
if e == None:
return Null
elif isinstance(e, (list, Except)):
return e
elif is_data(e):
e.cause = unwraplist([Except.wrap(c) for c in listwrap(e.cause)])
return Except(**e)
else:
tb = getattr(e, '__traceback__', None)
if tb is not None:
trace = _parse_traceback(tb)
else:
trace = _extract_traceback(0)
cause = Except.wrap(getattr(e, '__cause__', None))
if hasattr(e, "message") and e.message:
output = Except(context=ERROR, template=text_type(e.message), trace=trace, cause=cause)
else:
output = Except(context=ERROR, template=text_type(e), trace=trace, cause=cause)
trace = extract_stack(stack_depth + 2) # +2 = to remove the caller, and it's call to this' Except.wrap()
output.trace.extend(trace)
return output
def utf82unicode(value):
"""
WITH EXPLANATION FOR FAILURE
"""
try:
return value.decode("utf8")
except Exception as e:
if not _Log:
_late_import()
if not is_binary(value):
_Log.error("Can not convert {{type}} to unicode because it's not bytes", type= type(value).__name__)
e = _Except.wrap(e)
for i, c in enumerate(value):
try:
c.decode("utf8")
except Exception as f:
_Log.error("Can not convert charcode {{c}} in string index {{i}}", i=i, c=ord(c), cause=[e, _Except.wrap(f)])
try:
latin1 = text_type(value.decode("latin1"))
_Log.error("Can not explain conversion failure, but seems to be latin1", e)
except Exception:
pass
try:
a = text_type(value.decode("latin1"))
_Log.error("Can not explain conversion failure, but seems to be latin1", e)
except Exception:
pass
_Log.error("Can not explain conversion failure of " + type(value).__name__ + "!", e)
def _make_range_domain(self, domain, column_name):
width = (domain.max - domain.min) / domain.interval
digits = mo_math.floor(mo_math.log10(width - 1))
if digits == 0:
value = "a.value"
else:
value = SQL("+").join("1" + ("0" * j) + "*" +
text_type(chr(ord(b'a') + j)) + ".value"
for j in range(digits + 1))
if domain.interval == 1:
if domain.min == 0:
domain = (SQL_SELECT + value + column_name + SQL_FROM +
"__digits__ a")
else:
domain = (SQL_SELECT + sql_iso(value) + " + " +
quote_value(domain.min) + column_name + SQL_FROM +
"__digits__ a")
else:
if domain.min == 0:
domain = (SQL_SELECT + value + " * " +
quote_value(domain.interval) + column_name +
SQL_FROM + "__digits__ a")
else:
domain = (
SQL_SELECT +
sql_iso(value + " * " + quote_value(domain.interval)) +
" + " + quote_value(domain.min) + column_name + SQL_FROM +
"__digits__ a")
for j in range(digits):
domain += SQL_INNER_JOIN + "__digits__" + text_type(
chr(ord(b'a') + j + 1)) + " ON " + SQL_TRUE
domain += SQL_WHERE + value + " < " + quote_value(width)
return domain
def note(template, **params):
if not is_text(template):
Log.error("Log.note was expecting a unicode template")
if len(template) > 10000:
template = template[:10000]
log_params = wrap(
{
"template": template,
"params": params,
"timestamp": datetime.utcnow(),
"machine": machine_metadata,
"context": exceptions.NOTE,
}
)
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if Log.trace:
log_template = (
'{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - "{{location.file}}:{{location.line}}" ({{location.method}}) - '
+ template.replace("{{", "{{params.")
)
f = sys._getframe(1)
log_params.location = {
"line": f.f_lineno,
"file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
"method": text_type(f.f_code.co_name),
}
else:
log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.")
Log.main_log.write(log_template, log_params)
def note(template, **params):
if not isinstance(template, text_type):
Log.error("Log.note was expecting a unicode template")
if len(template) > 10000:
template = template[:10000]
log_params = wrap({
"template": template,
"params": params,
"timestamp": datetime.utcnow(),
"machine": machine_metadata,
"context": exceptions.NOTE
})
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if Log.trace:
log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace(
"{{", "{{params.")
f = sys._getframe(1)
log_params.location = {
"line": f.f_lineno,
"file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
"method": text_type(f.f_code.co_name)
}
else:
log_template = "{{timestamp|datetime}} - " + template.replace(
"{{", "{{params.")
Log.main_log.write(log_template, log_params)
def to_python(self, not_null=False, boolean=False, many=False):
return (
"row["
+ text_type(self.var)
+ "] if 0<="
+ text_type(self.var)
+ "<len(row) else None"
)
def sort_table(result):
"""
SORT ROWS IN TABLE, EVEN IF ELEMENTS ARE JSON
"""
data = wrap([{text_type(i): v for i, v in enumerate(row)} for row in result.data])
sort_columns = jx.sort(set(jx.get_columns(data, leaves=True).name))
data = jx.sort(data, sort_columns)
result.data = [tuple(row[text_type(i)] for i in range(len(result.header))) for row in data]
def tab(value):
if isinstance(value, Mapping):
h, d = zip(*wrap(value).leaves())
return \
"\t".join(map(value2json, h)) + \
"\n" + \
"\t".join(map(value2json, d))
else:
text_type(value)
def tab(value):
"""
convert single value to tab-delimited form, including a header
:param value:
:return:
"""
if isinstance(value, Mapping):
h, d = zip(*wrap(value).leaves())
return ("\t".join(map(value2json, h)) + "\n" +
"\t".join(map(value2json, d)))
else:
text_type(value)
def tab(value):
"""
convert single value to tab-delimited form, including a header
:param value:
:return:
"""
if is_data(value):
h, d = transpose(*wrap(value).leaves())
return ("\t".join(map(value2json, h)) + CR +
"\t".join(map(value2json, d)))
else:
text_type(value)
def box(script):
"""
:param es_script:
:return: TEXT EXPRESSION WITH NON OBJECTS BOXED
"""
if script.type is BOOLEAN:
return "Boolean.valueOf(" + text_type(script.expr) + ")"
elif script.type is INTEGER:
return "Integer.valueOf(" + text_type(script.expr) + ")"
elif script.type is NUMBER:
return "Double.valueOf(" + text_type(script.expr) + ")"
else:
return script.expr
def box(script):
"""
:param es_script:
:return: TEXT EXPRESSION WITH NON OBJECTS BOXED
"""
if script.type is BOOLEAN:
return "Boolean.valueOf(" + text_type(script.expr) + ")"
elif script.type is INTEGER:
return "Integer.valueOf(" + text_type(script.expr) + ")"
elif script.type is NUMBER:
return "Double.valueOf(" + text_type(script.expr) + ")"
else:
return script.expr
def _convert(v):
if v is None:
return null_script
if v is True:
return true_script
if v is False:
return false_script
class_ = v.__class__
if class_ is text_type:
return EsScript(type=STRING,
expr=quote(v),
frum=self,
schema=schema)
if class_ in integer_types:
if MIN_INT32 <= v <= MAX_INT32:
return EsScript(type=INTEGER,
expr=text_type(v),
frum=self,
schema=schema)
else:
return EsScript(type=INTEGER,
expr=text_type(v) + "L",
frum=self,
schema=schema)
if class_ is float:
return EsScript(type=NUMBER,
expr=text_type(v) + "D",
frum=self,
schema=schema)
if class_ in data_types:
return EsScript(
type=OBJECT,
expr="[" + ", ".join(
quote(k) + ": " + _convert(vv)
for k, vv in v.items()) + "]",
frum=self,
schema=schema,
)
if class_ in (FlatList, list, tuple):
return EsScript(
type=OBJECT,
expr="[" + ", ".join(_convert(vv).expr for vv in v) + "]",
frum=self,
schema=schema,
)
if class_ is Date:
return EsScript(type=NUMBER,
expr=text_type(v.unix),
frum=self,
schema=schema)
def _convert(v):
if v is None:
return NULL.to_es_script(schema)
if v is True:
return EsScript(
type=BOOLEAN,
expr="true",
frum=self
)
if v is False:
return EsScript(
type=BOOLEAN,
expr="false",
frum=self
)
if isinstance(v, text_type):
return EsScript(
type=STRING,
expr=quote(v),
frum=self
)
if isinstance(v, int):
return EsScript(
type=INTEGER,
expr=text_type(v),
frum=self
)
if isinstance(v, float):
return EsScript(
type=NUMBER,
expr=text_type(v),
frum=self
)
if isinstance(v, dict):
return EsScript(
type=OBJECT,
expr="[" + ", ".join(quote(k) + ": " + _convert(vv) for k, vv in v.items()) + "]",
frum=self
)
if isinstance(v, (list, tuple)):
return EsScript(
type=OBJECT,
expr="[" + ", ".join(_convert(vv).expr for vv in v) + "]",
frum=self
)
if isinstance(v, Date):
return EsScript(
type=NUMBER,
expr=text_type(v.unix),
frum=self
)
def table2csv(table_data):
"""
:param table_data: expecting a list of tuples
:return: text in nice formatted csv
"""
text_data = [tuple(value2json(vals, pretty=True) for vals in rows) for rows in table_data]
col_widths = [max(len(text) for text in cols) for cols in zip(*text_data)]
template = ", ".join(
"{{" + text_type(i) + "|left_align(" + text_type(w) + ")}}"
for i, w in enumerate(col_widths)
)
text = "\n".join(expand_template(template, d) for d in text_data)
return text
def encode(self, value, pretty=False):
if pretty:
return pretty_json(value)
try:
scrubbed = scrub(value)
return text_type(self.encoder(scrubbed))
except Exception as e:
from mo_logs.exceptions import Except
from mo_logs import Log
e = Except.wrap(e)
Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e)
raise e
def table2csv(table_data):
"""
:param table_data: expecting a list of tuples
:return: text in nice formatted csv
"""
text_data = [tuple(value2json(vals, pretty=True) for vals in rows) for rows in table_data]
col_widths = [max(len(text) for text in cols) for cols in zip(*text_data)]
template = ", ".join(
"{{" + text_type(i) + "|left_align(" + text_type(w) + ")}}"
for i, w in enumerate(col_widths)
)
text = "\n".join(expand_template(template, d) for d in text_data)
return text
def note(cls,
template,
default_params={},
stack_depth=0,
log_context=None,
**more_params):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if not isinstance(template, text_type):
Log.error("Log.note was expecting a unicode template")
if len(template) > 10000:
template = template[:10000]
params = dict(unwrap(default_params), **more_params)
log_params = set_default(
{
"template": template,
"params": params,
"timestamp": datetime.utcnow(),
"machine": machine_metadata
}, log_context, {"context": exceptions.NOTE})
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if cls.trace:
log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace(
"{{", "{{params.")
f = sys._getframe(stack_depth + 1)
log_params.location = {
"line": f.f_lineno,
"file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
"method": text_type(f.f_code.co_name)
}
thread = _Thread.current()
log_params.thread = {"name": thread.name, "id": thread.id}
else:
log_template = "{{timestamp|datetime}} - " + template.replace(
"{{", "{{params.")
cls.main_log.write(log_template, log_params)
def tab(value):
"""
convert single value to tab-delimited form, including a header
:param value:
:return:
"""
if is_data(value):
h, d = transpose(*wrap(value).leaves())
return (
"\t".join(map(value2json, h)) +
CR +
"\t".join(map(value2json, d))
)
else:
text_type(value)
def note(
cls,
template,
default_params={},
stack_depth=0,
log_context=None,
**more_params
):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if not isinstance(template, text_type):
Log.error("Log.note was expecting a unicode template")
if len(template) > 10000:
template = template[:10000]
params = dict(unwrap(default_params), **more_params)
log_params = set_default({
"template": template,
"params": params,
"timestamp": datetime.utcnow(),
"machine": machine_metadata
}, log_context, {"context": exceptions.NOTE})
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if cls.trace:
log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace("{{", "{{params.")
f = sys._getframe(stack_depth + 1)
log_params.location = {
"line": f.f_lineno,
"file": text_type(f.f_code.co_filename.split(os.sep)[-1]),
"method": text_type(f.f_code.co_name)
}
thread = _Thread.current()
log_params.thread = {"name": thread.name, "id": thread.id}
else:
log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.")
cls.main_log.write(log_template, log_params)
def __init__(self, ident):
self.id = ident
if ident != -1:
self.name = "Unknown Thread " + text_type(ident)
self.child_lock = allocate_lock()
self.children = []
self.cprofiler = None
def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None):
self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD)
self.rate = coalesce(rate, HG_REQUEST_PER_SECOND)
self.cache_locker = Lock()
self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR
self.no_cache = {} # VERY SHORT TERM CACHE
self.workers = []
self.todo = Queue(APP_NAME+" todo")
self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds))
self.url = URL(source.url)
self.db = Sqlite(database)
self.inbound_rate = RateLogger("Inbound")
self.outbound_rate = RateLogger("hg.mo")
if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data:
with self.db.transaction() as t:
t.execute(
"CREATE TABLE cache ("
" path TEXT PRIMARY KEY, "
" headers TEXT, "
" response TEXT, "
" timestamp REAL "
")"
)
self.threads = [
Thread.run(APP_NAME+" worker" + text_type(i), self._worker)
for i in range(CONCURRENCY)
]
self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter)
self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def __getitem__(self, key):
if key == None:
return Null
if key == ".":
output = self._internal_dict
if isinstance(output, Mapping):
return self
else:
return output
key = text_type(key)
d = self._internal_dict
if key.find(".") >= 0:
seq = _split_field(key)
for n in seq:
if isinstance(d, NullType):
d = NullType(d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL
elif isinstance(d, list):
d = [_getdefault(dd, n) for dd in d]
else:
d = _getdefault(d, n) # EVERYTHING ELSE TREATS n AS LITERAL
return wrap(d)
else:
o = d.get(key)
if o == None:
return NullType(d, key)
return wrap(o)
def float2json(value):
"""
CONVERT NUMBER TO JSON STRING, WITH BETTER CONTROL OVER ACCURACY
:param value: float, int, long, Decimal
:return: unicode
"""
if value == 0:
return u'0'
try:
sign = "-" if value < 0 else ""
value = abs(value)
sci = value.__format__(".15e")
mantissa, exp = sci.split("e")
exp = int(exp)
if 0 <= exp:
digits = u"".join(mantissa.split("."))
return sign + (digits[:1 + exp] + u"." +
digits[1 + exp:].rstrip('0')).rstrip(".")
elif -4 < exp:
digits = ("0" * (-exp)) + u"".join(mantissa.split("."))
return sign + (digits[:1] + u"." +
digits[1:].rstrip('0')).rstrip(".")
else:
return sign + mantissa.rstrip("0") + u"e" + text_type(exp)
except Exception as e:
from mo_logs import Log
Log.error("not expected", e)
def value2json(obj, pretty=False, sort_keys=False, keep_whitespace=True):
"""
:param obj: THE VALUE TO TURN INTO JSON
:param pretty: True TO MAKE A MULTI-LINE PRETTY VERSION
:param sort_keys: True TO SORT KEYS
:param keep_whitespace: False TO strip() THE WHITESPACE IN THE VALUES
:return:
"""
if FIND_LOOPS:
obj = scrub(obj,
scrub_text=_keep_whitespace
if keep_whitespace else _trim_whitespace())
try:
json = json_encoder(obj, pretty=pretty)
if json == None:
Log.note(str(type(obj)) + " is not valid{{type}}JSON",
type=" (pretty) " if pretty else " ")
Log.error("Not valid JSON: " + str(obj) + " of type " +
str(type(obj)))
return json
except Exception as e:
e = Except.wrap(e)
try:
json = pypy_json_encode(obj)
return json
except Exception:
pass
Log.error("Can not encode into JSON: {{value}}",
value=text_type(repr(obj)),
cause=e)
def __getitem__(self, key):
if key == None:
return Null
if key == ".":
output = self._internal_dict
if _get(output, CLASS) in data_types:
return self
else:
return output
key = text_type(key)
d = self._internal_dict
if key.find(".") >= 0:
seq = _split_field(key)
for n in seq:
if _get(d, CLASS) is NullType:
d = NullType(
d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL
elif is_list(d):
d = [_getdefault(dd, n) for dd in d]
else:
d = _getdefault(d,
n) # EVERYTHING ELSE TREATS n AS LITERAL
return wrap(d)
else:
o = d.get(key)
if o == None:
return NullType(d, key)
return wrap(o)
def encode(self, value, pretty=False):
if pretty:
return pretty_json(value)
try:
with Timer("scrub", too_long=0.1):
scrubbed = scrub(value)
with Timer("encode", too_long=0.1):
return text_type(self.encoder(scrubbed))
except Exception as e:
from mo_logs.exceptions import Except
from mo_logs import Log
e = Except.wrap(e)
Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e)
raise e
def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None):
self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD)
self.rate = coalesce(rate, HG_REQUEST_PER_SECOND)
self.cache_locker = Lock()
self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR
self.no_cache = {} # VERY SHORT TERM CACHE
self.workers = []
self.todo = Queue(APP_NAME+" todo")
self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds))
self.url = URL(source.url)
self.db = Sqlite(database)
self.inbound_rate = RateLogger("Inbound")
self.outbound_rate = RateLogger("hg.mo")
if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data:
with self.db.transaction() as t:
t.execute(
"CREATE TABLE cache ("
" path TEXT PRIMARY KEY, "
" headers TEXT, "
" response TEXT, "
" timestamp REAL "
")"
)
self.threads = [
Thread.run(APP_NAME+" worker" + text_type(i), self._worker)
for i in range(CONCURRENCY)
]
self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter)
self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def main():
try:
settings = startup.read_settings()
with startup.SingleInstance(settings.args.filename):
constants.set(settings.constants)
Log.start(settings.debug)
extractor = Extract(settings)
def extract(please_stop):
with MySQL(**settings.snowflake.database) as db:
with db.transaction():
for kwargs in extractor.queue:
if please_stop:
break
try:
extractor.extract(db=db,
please_stop=please_stop,
**kwargs)
except Exception as e:
Log.warning("Could not extract", cause=e)
extractor.queue.add(kwargs)
for i in range(settings.extract.threads):
Thread.run("extract #" + text_type(i), extract)
please_stop = Signal()
Thread.wait_for_shutdown_signal(please_stop=please_stop,
allow_exit=True,
wait_forever=False)
except Exception as e:
Log.warning("Problem with data extraction", e)
finally:
Log.stop()
def test_single_file_list(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.post_json(
url,
json={
"meta": {
"format": "list"
},
"from": "files",
"where": {
"and": [{
"eq": {
"revision": "29dcc9cb77c372c97681a47496488ec6c623915d"
}
}, {
"in": {
"path": ["gfx/thebes/gfxFontVariations.h"]
}
}, {
"eq": {
"branch": "mozilla-central"
}
}]
}
})
list_response = response.data
tuids = list_response[0].tuids
assert len(tuids) == 41 # 41 lines expected
assert len(set(tuids)) == 41 # tuids much be unique
def get(url):
"""
USE json.net CONVENTIONS TO LINK TO INLINE OTHER JSON
"""
url = text_type(url)
if url.find("://") == -1:
Log.error("{{url}} must have a prototcol (eg http://) declared",
url=url)
base = URL("")
if url.startswith("file://") and url[7] != "/":
if os.sep == "\\":
base = URL("file:///" +
os.getcwd().replace(os.sep, "/").rstrip("/") + "/.")
else:
base = URL("file://" + os.getcwd().rstrip("/") + "/.")
elif url[url.find("://") + 3] != "/":
Log.error("{{url}} must be absolute", url=url)
phase1 = _replace_ref(wrap({"$ref": url}),
base) # BLANK URL ONLY WORKS IF url IS ABSOLUTE
try:
phase2 = _replace_locals(phase1, [phase1])
return wrap(phase2)
except Exception as e:
Log.error("problem replacing locals in\n{{phase1}}",
phase1=phase1,
cause=e)
def format(self, format="%Y-%m-%d %H:%M:%S"):
try:
return text_type(unix2datetime(self.unix).strftime(format))
except Exception as e:
from mo_logs import Log
Log.error("Can not format {{value}} with {{format}}", value=unix2datetime(self.unix), format=format, cause=e)
def __init__(self, ident):
self.id = ident
if ident != -1:
self.name = "Unknown Thread " + text_type(ident)
self.child_lock = allocate_lock()
self.children = []
self.cprofiler = None
def _find_revision(self, revision):
please_stop = False
locker = Lock()
output = []
queue = Queue("branches", max=2000)
queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"])
queue.add(THREAD_STOP)
problems = []
def _find(please_stop):
for b in queue:
if please_stop:
return
try:
url = b.url + "json-info?node=" + revision
rev = self.get_revision(Revision(branch=b, changeset={"id": revision}))
with locker:
output.append(rev)
Log.note("Revision found at {{url}}", url=url)
except Exception as f:
problems.append(f)
threads = []
for i in range(3):
threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop))
for t in threads:
with assert_no_exception:
t.join()
return output
def queue_consumer(pull_queue, please_stop=None):
queue = aws.Queue(pull_queue)
time_offset = None
request_count = 0
while not please_stop:
request = queue.pop(till=please_stop)
if please_stop:
break
if not request:
Log.note("Nothing in queue, pausing for 5 seconds...")
(please_stop | Till(seconds=5)).wait()
continue
if SKIP_TRY_REQUESTS and 'try' in request.where['and'].eq.branch:
Log.note("Skipping try revision.")
queue.commit()
continue
now = Date.now().unix
if time_offset is None:
time_offset = now - request.meta.request_time
next_request = request.meta.request_time + time_offset
if next_request > now:
Log.note("Next request in {{wait_time}}", wait_time=Duration(seconds=next_request - now))
Till(till=next_request).wait()
Thread.run("request "+text_type(request_count), one_request, request)
request_count += 1
queue.commit()
def problem_serializing(value, e=None):
"""
THROW ERROR ABOUT SERIALIZING
"""
from mo_logs import Log
try:
typename = type(value).__name__
except Exception:
typename = "<error getting name>"
try:
rep = text_type(repr(value))
except Exception as _:
rep = None
if rep == None:
Log.error(
"Problem turning value of type {{type}} to json",
type=typename,
cause=e
)
else:
Log.error(
"Problem turning value ({{value}}) of type {{type}} to json",
value=rep,
type=typename,
cause=e
)
def latin12unicode(value):
if isinstance(value, text_type):
Log.error("can not convert unicode from latin1")
try:
return text_type(value.decode('latin1'))
except Exception as e:
Log.error("Can not convert {{value|quote}} to unicode", value=value)
def to_es_script(self, schema):
return EsScript(
type=NUMBER,
expr=text_type(Date(self.value).unix),
frum=self,
schema=schema
)
def problem_serializing(value, e=None):
"""
THROW ERROR ABOUT SERIALIZING
"""
from mo_logs import Log
try:
typename = type(value).__name__
except Exception:
typename = "<error getting name>"
try:
rep = text_type(repr(value))
except Exception as _:
rep = None
if rep == None:
Log.error("Problem turning value of type {{type}} to json",
type=typename,
cause=e)
else:
Log.error("Problem turning value ({{value}}) of type {{type}} to json",
value=rep,
type=typename,
cause=e)
def queue_consumer(pull_queue, please_stop=None):
queue = aws.Queue(pull_queue)
time_offset = None
request_count = 0
while not please_stop:
request = queue.pop(till=please_stop)
if please_stop:
break
if not request:
Log.note("Nothing in queue, pausing for 5 seconds...")
(please_stop | Till(seconds=5)).wait()
continue
if SKIP_TRY_REQUESTS and 'try' in request.where['and'].eq.branch:
Log.note("Skipping try revision.")
queue.commit()
continue
now = Date.now().unix
if time_offset is None:
time_offset = now - request.meta.request_time
next_request = request.meta.request_time + time_offset
if next_request > now:
Log.note("Next request in {{wait_time}}",
wait_time=Duration(seconds=next_request - now))
Till(till=next_request).wait()
Thread.run("request " + text_type(request_count), one_request, request)
request_count += 1
queue.commit()
def _find_revision(self, revision):
please_stop = False
locker = Lock()
output = []
queue = Queue("branches", max=2000)
queue.extend(b for b in self.branches if b.locale == DEFAULT_LOCALE and b.name in ["try", "mozilla-inbound", "autoland"])
queue.add(THREAD_STOP)
problems = []
def _find(please_stop):
for b in queue:
if please_stop:
return
try:
url = b.url + "json-info?node=" + revision
rev = self.get_revision(Revision(branch=b, changeset={"id": revision}))
with locker:
output.append(rev)
Log.note("Revision found at {{url}}", url=url)
except Exception as f:
problems.append(f)
threads = []
for i in range(3):
threads.append(Thread.run("find changeset " + text_type(i), _find, please_stop=please_stop))
for t in threads:
with assert_no_exception:
t.join()
return output
def quote_value(self, value):
"""
convert values to mysql code for the same
mostly delegate directly to the mysql lib, but some exceptions exist
"""
try:
if value == None:
return SQL_NULL
elif isinstance(value, SQL):
if not value.param:
# value.template CAN BE MORE THAN A TEMPLATE STRING
return self.quote_sql(value.template)
param = {k: self.quote_sql(v) for k, v in value.param.items()}
return SQL(expand_template(value.template, param))
elif isinstance(value, text_type):
return SQL(self.db.literal(value))
elif isinstance(value, Mapping):
return SQL(self.db.literal(json_encode(value)))
elif Math.is_number(value):
return SQL(text_type(value))
elif isinstance(value, datetime):
return SQL("str_to_date('" +
value.strftime("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif isinstance(value, Date):
return SQL("str_to_date('" + value.format("%Y%m%d%H%M%S.%f") +
"', '%Y%m%d%H%i%s.%f')")
elif hasattr(value, '__iter__'):
return SQL(self.db.literal(json_encode(value)))
else:
return self.db.literal(value)
except Exception as e:
Log.error("problem quoting SQL", e)
def __getitem__(self, key):
if key == None:
return Null
if key == ".":
output = _get(self, "_dict")
if isinstance(output, Mapping):
return self
else:
return output
key = text_type(key)
d = _get(self, "_dict")
if key.find(".") >= 0:
seq = _split_field(key)
for n in seq:
if isinstance(d, NullType):
d = NullType(
d, n) # OH DEAR, Null TREATS n AS PATH, NOT LITERAL
elif isinstance(d, list):
d = [_getdefault(dd, n) for dd in d]
else:
d = _getdefault(d,
n) # EVERYTHING ELSE TREATS n AS LITERAL
return wrap(d)
else:
o = d.get(key)
if o == None:
return NullType(d, key)
return wrap(o)
def unexpected(
cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params
):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
if cause and not isinstance(cause, Except):
cause = Except(exceptions.UNEXPECTED, text_type(cause), trace=exceptions._extract_traceback(0))
trace = exceptions.extract_stack(1)
e = Except(type=exceptions.UNEXPECTED, template=template, params=params, cause=cause, trace=trace)
Log.note(
"{{error}}",
error=e,
log_context=set_default({"context": exceptions.WARNING}, log_context),
stack_depth=stack_depth + 1
)
def write(self, template, params):
try:
params.template = strings.limit(params.template, 2000)
params.format = None
self.queue.add({"value": _deep_json_to_string(params, 3)}, timeout=3 * 60)
except Exception as e:
sys.stdout.write(text_type(Except.wrap(e)))
return self
def unicode_key(key):
"""
CONVERT PROPERTY VALUE TO QUOTED NAME OF SAME
"""
if not isinstance(key, (text_type, binary_type)):
from mo_logs import Log
Log.error("{{key|quote}} is not a valid key", key=key)
return quote(text_type(key))
def to_es(self, schema, query_path="."):
if self.children:
return {"aggs": {
name: t.to_es(schema, query_path)
for i, t in enumerate(self.children)
for name in [t.name if t.name else "_" + text_type(i)]
}}
else:
return {}
def __setattr__(self, key, value):
key = text_type(key)
d = _get(self, "__dict__")
o = wrap(d["_obj"])
k = d["__key__"]
seq = [k] + [key]
_assign_to_null(o, seq, value)
def unicode(value):
"""
Convert to a unicode string
:param value: any value
:return: unicode
"""
if value == None:
return ""
return text_type(value)
def index_to_coordinate(dims):
"""
RETURN A FUNCTION THAT WILL TAKE AN INDEX, AND MAP IT TO A coordinate IN dims
:param dims: TUPLE WITH NUMBER OF POINTS IN EACH DIMENSION
:return: FUNCTION
"""
_ = divmod # SO WE KEEP THE IMPORT
num_dims = len(dims)
if num_dims == 0:
return _zero_dim
prod = [1] * num_dims
acc = 1
domain = range(0, num_dims)
for i in reversed(domain):
prod[i] = acc
acc *= dims[i]
commands = []
coords = []
for i in domain:
if i == num_dims - 1:
commands.append("\tc" + text_type(i) + " = index")
else:
commands.append("\tc" + text_type(i) + ", index = divmod(index, " + text_type(prod[i]) + ")")
coords.append("c" + text_type(i))
output = None
if num_dims == 1:
code = (
"def output(index):\n" +
"\n".join(commands) + "\n" +
"\treturn " + coords[0] + ","
)
else:
code = (
"def output(index):\n" +
"\n".join(commands) + "\n" +
"\treturn " + ", ".join(coords)
)
exec(code)
return output
def append_query(self, query_path, es_query):
domain = self.domain
domain_key = domain.key
value = Painless[self.edge.value]
cnv = pull_functions[value.type]
include = tuple(cnv(p[domain_key]) for p in domain.partitions)
exists = Painless[AndOp([
InOp([value, Literal(include)])
])].partial_eval()
limit = coalesce(self.limit, len(domain.partitions))
if is_op(value, Variable):
es_field = first(self.query.frum.schema.leaves(value.var)).es_column # ALREADY CHECKED THERE IS ONLY ONE
match = TermsAggs(
"_match",
{
"field": es_field,
"size": limit,
"order": {"_term": self.sorted} if self.sorted else None
},
self
)
else:
match = TermsAggs(
"_match",
{
"script": text_type(value.to_es_script(self.schema)),
"size": limit
},
self
)
output = Aggs().add(FilterAggs("_filter", exists, None).add(match.add(es_query)))
if self.edge.allowNulls:
# FIND NULLS AT EACH NESTED LEVEL
for p in self.schema.query_path:
if p == query_path:
# MISSING AT THE QUERY DEPTH
output.add(
NestedAggs(p).add(FilterAggs("_missing0", NotOp(exists), self).add(es_query))
)
else:
# PARENT HAS NO CHILDREN, SO MISSING
column = first(self.schema.values(query_path, (OBJECT, EXISTS)))
output.add(
NestedAggs(column.nested_path[0]).add(
FilterAggs(
"_missing1",
NotOp(ExistsOp(Variable(column.es_column.replace(NESTED_TYPE, EXISTS_TYPE)))),
self
).add(es_query)
)
)
return output
def encode(self, value, pretty=False):
if pretty:
return pretty_json(value)
try:
with Timer("scrub", too_long=0.1):
scrubbed = scrub(value)
param = {"size": 0}
with Timer("encode {{size}} characters", param=param, too_long=0.1):
output = text_type(self.encoder(scrubbed))
param["size"] = len(output)
return output
except Exception as e:
from mo_logs.exceptions import Except
from mo_logs import Log
e = Except.wrap(e)
Log.warning("problem serializing {{type}}", type=text_type(repr(value)), cause=e)
raise e
