def __init__(self,
name,
region,
aws_access_key_id=None,
aws_secret_access_key=None,
debug=False,
settings=None):
self.settings = settings
self.pending = []
if settings.region not in [r.name for r in sqs.regions()]:
Log.error("Can not find region {{region}} in {{regions}}",
region=settings.region,
regions=[r.name for r in sqs.regions()])
conn = sqs.connect_to_region(
region_name=unwrap(settings.region),
aws_access_key_id=unwrap(settings.aws_access_key_id),
aws_secret_access_key=unwrap(settings.aws_secret_access_key),
)
self.queue = conn.get_queue(settings.name)
if self.queue == None:
Log.error(
"Can not find queue with name {{queue}} in region {{region}}",
queue=settings.name,
region=settings.region)
def __setitem__(self, key, value):
if key == "":
from pyLibrary.debugs.logs import Log
Log.error("key is empty string. Probably a bad idea")
if key == ".":
# SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping;
# HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS unwrap()
v = unwrap(value)
_set(self, "_dict", v)
return v
if isinstance(key, str):
key = key.decode("utf8")
try:
d = _get(self, "_dict")
value = unwrap(value)
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
return self
seq = _split_field(key)
for k in seq[:-1]:
d = _getdefault(d, k)
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return self
except Exception, e:
raise e
def __setitem__(self, key, value):
if key == "":
from pyLibrary.debugs.logs import Log
Log.error("key is empty string. Probably a bad idea")
if key == ".":
# SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping;
# HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS unwrap()
v = unwrap(value)
_set(self, "_dict", v)
return v
if isinstance(key, str):
key = key.decode("utf8")
try:
d = _get(self, "_dict")
value = unwrap(value)
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
return self
seq = _split_field(key)
for k in seq[:-1]:
d = _getdefault(d, k)
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return self
except Exception, e:
raise e
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
aws_args = dict(
region_name=settings.aws.region,
aws_access_key_id=unwrap(settings.aws.aws_access_key_id),
aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key)
)
ec2_conn = boto_ec2.connect_to_region(**aws_args)
instances = _get_managed_instances(ec2_conn, settings.name)
for i in instances:
Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", insance_id=i.id, name=i.tags["Name"], ip=i.ip_address)
_config_fabric(settings.fabric, i)
try:
_refresh_etl() # TODO: UPON FAILURE, TERMINATE INSTANCE AND SPOT REQUEST
except Exception, e:
ec2_conn.terminate_instances([i.id])
Log.warning("Problem resetting {{instance}}, terminated", instance=i.id, cause=e)
except Exception, e:
Log.error("Problem with etl", e)
def filter(data, where):
"""
where - a function that accepts (record, rownum, rows) and returns boolean
"""
if len(data) == 0 or where == None or where == TRUE_FILTER:
return data
if isinstance(data, Container):
return data.filter(where)
if isinstance(data, (list, set)):
temp = jx_expression_to_function(where)
dd = wrap(data)
return wrap(
[unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)])
else:
Log.error("Do not know how to handle type {{type}}",
type=data.__class__.__name__)
try:
return drill_filter(where, data)
except Exception, _:
# WOW! THIS IS INEFFICIENT!
return wrap([
unwrap(d)
for d in drill_filter(where, [DictObject(d) for d in data])
])
def es_fieldop(es, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
FromES.query = {
"filtered": {
"query": {
"match_all": {}
},
"filter":
simplify_esfilter(jx_expression(query.where).to_esfilter())
}
}
FromES.size = coalesce(query.limit, 200000)
FromES.fields = DictList()
for s in select.value:
if s == "*":
FromES.fields = None
elif isinstance(s, list):
FromES.fields.extend(s)
elif isinstance(s, Mapping):
FromES.fields.extend(s.values())
else:
FromES.fields.append(s)
FromES.sort = [{
s.field: "asc" if s.sort >= 0 else "desc"
} for s in query.sort]
data = es09.util.post(es, FromES, query.limit)
T = data.hits.hits
matricies = {}
for s in select:
if s.value == "*":
matricies[s.name] = Matrix.wrap([t._source for t in T])
elif isinstance(s.value, Mapping):
# for k, v in s.value.items():
# matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T])
matricies[s.name] = Matrix.wrap([{
k: unwrap(t.fields).get(v, None)
for k, v in s.value.items()
} for t in T])
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([
tuple(unwrap(t.fields).get(ss, None) for ss in s.value)
for t in T
])
elif not s.value:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
else:
try:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
except Exception, e:
Log.error("", e)
def extend(self, records):
"""
JUST SO WE MODEL A Queue
"""
records = {v["id"]: v["value"] for v in records}
unwrap(self.data).update(records)
data_as_json = convert.value2json(self.data, pretty=True)
File(self.filename).write(data_as_json)
Log.note("{{num}} documents added", num= len(records))
def extend(self, records):
"""
JUST SO WE MODEL A Queue
"""
records = {v["id"]: v["value"] for v in records}
unwrap(self.data).update(records)
data_as_json = convert.value2json(self.data, pretty=True)
File(self.filename).write(data_as_json)
Log.note("{{num}} documents added", num=len(records))
def warning(cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
if "values" in more_params.keys():
Log.error("Can not handle a logging parameter by name `values`")
params = dict(unwrap(default_params), **more_params)
cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
trace = exceptions.extract_stack(stack_depth + 1)
e = Except(exceptions.WARNING, template, params, cause, trace)
Log.note("{{error|unicode}}",
error=e,
log_context=set_default({"context": exceptions.WARNING},
log_context),
stack_depth=stack_depth + 1)
def dict2Multiset(dic):
if dic == None:
return None
output = Multiset()
output.dic = unwrap(dic).copy()
return output
def __setitem__(self, key, value):
if key == "":
from pyLibrary.debugs.logs import Log
Log.error("key is empty string. Probably a bad idea")
if isinstance(key, str):
key = key.decode("utf8")
d=self
try:
value = unwrap(value)
if key.find(".") == -1:
if value is None:
dict.pop(d, key, None)
else:
dict.__setitem__(d, key, value)
return self
seq = _split_field(key)
for k in seq[:-1]:
d = _getdefault(d, k)
if value == None:
dict.pop(d, seq[-1], None)
else:
dict.__setitem__(d, seq[-1], value)
return self
except Exception, e:
raise e
def error(
cls,
template, # human readable template
default_params={}, # parameters for template
cause=None, # pausible cause
stack_depth=0,
**more_params
):
"""
raise an exception with a trace for the cause too
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if default_params and isinstance(listwrap(default_params)[0], BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
add_to_trace = False
cause = wrap(unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)]))
trace = exceptions.extract_stack(stack_depth + 1)
if add_to_trace:
cause[0].trace.extend(trace[1:])
e = Except(exceptions.ERROR, template, params, cause, trace)
raise e
def warning(
cls,
template,
default_params={},
cause=None,
stack_depth=0, # stack trace offset (==1 if you do not want to report self)
**more_params
):
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
trace = extract_stack(stack_depth + 1)
e = Except(WARNING, template, params, cause, trace)
Log.note(
unicode(e),
{
"warning": {# REDUNDANT INFO
"template": template,
"params": params,
"cause": cause,
"trace": trace
}
},
stack_depth=stack_depth + 1
)
def test_request(self):
# MAKE SOME DATA
data = {
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
client = Client(settings.url, unwrap(settings.hawk)) # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21
link, id = client.send(data)
Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id)
# FILL THE REST OF THE FILE
Log.note("Add ing {{num}} more...", num=99-id)
for i in range(id + 1, storage.BATCH_SIZE):
l, k = client.send(data)
if l != link:
Log.error("Expecting rest of data to have same link")
# TEST LINK HAS DATA
raw_content = requests.get(link).content
content = convert.zip2bytes(raw_content)
for line in convert.utf82unicode(content).split("\n"):
data = convert.json2value(line)
if data.etl.id == id:
Log.note("Data {{id}} found", id=id)
break
else:
Log.error("Expecting to find data at link")
def note(cls, template, default_params={}, stack_depth=0, **more_params):
if len(template) > 10000:
template = template[:10000]
params = dict(unwrap(default_params), **more_params)
log_params = Dict(template=template,
params=params,
timestamp=datetime.utcnow(),
machine=machine_metadata.name)
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if cls.trace:
log_template = "{{machine}} - {{timestamp|datetime}} - {{thread.name}} - {{location.file}}:{{location.line}} ({{location.method}}) - " + template.replace(
"{{", "{{params.")
f = sys._getframe(stack_depth + 1)
log_params.location = {
"line": f.f_lineno,
"file": f.f_code.co_filename.split(os.sep)[-1],
"method": f.f_code.co_name
}
thread = Thread.current()
log_params.thread = {"name": thread.name, "id": thread.id}
else:
log_template = "{{timestamp|datetime}} - " + template.replace(
"{{", "{{params.")
cls.main_log.write(log_template, log_params)
def unexpected(cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
if cause and not isinstance(cause, Except):
cause = Except(exceptions.UNEXPECTED,
unicode(cause),
trace=exceptions._extract_traceback(0))
trace = exceptions.extract_stack(1)
e = Except(exceptions.UNEXPECTED, template, params, cause, trace)
Log.note("{{error}}",
error=e,
log_context=set_default({"context": exceptions.WARNING},
log_context),
stack_depth=stack_depth + 1)
def warning(
cls,
template,
default_params={},
cause=None,
stack_depth=0, # stack trace offset (==1 if you do not want to report self)
**more_params):
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
trace = extract_stack(stack_depth + 1)
e = Except(WARNING, template, params, cause, trace)
Log.note(
unicode(e),
{
"warning": { # REDUNDANT INFO
"template": template,
"params": params,
"cause": cause,
"trace": trace
}
},
stack_depth=stack_depth + 1)
def unexpected(cls,
template,
default_params={},
cause=None,
**more_params):
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
if cause and not isinstance(cause, Except):
cause = Except(UNEXPECTED, unicode(cause), trace=extract_tb(0))
trace = extract_stack(1)
e = Except(UNEXPECTED, template, params, cause, trace)
Log.note(
unicode(e), {
"warning": {
"template": template,
"params": params,
"cause": cause,
"trace": trace
}
})
def tuple(data, field_name):
"""
RETURN LIST OF TUPLES
"""
if isinstance(data, Cube):
Log.error("not supported yet")
if isinstance(data, FlatList):
Log.error("not supported yet")
if isinstance(field_name, Mapping) and "value" in field_name:
# SIMPLIFY {"value":value} AS STRING
field_name = field_name["value"]
# SIMPLE PYTHON ITERABLE ASSUMED
if isinstance(field_name, basestring):
if len(split_field(field_name)) == 1:
return [(d[field_name],) for d in data]
else:
path = split_field(field_name)
output = []
flat_list._tuple1(data, path, 0, output)
return output
elif isinstance(field_name, list):
paths = [_select_a_field(f) for f in field_name]
output = DictList()
_tuple((), unwrap(data), paths, 0, output)
return output
else:
paths = [_select_a_field(field_name)]
output = DictList()
_tuple((), data, paths, 0, output)
return output
def unexpected(
cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params
):
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
if cause and not isinstance(cause, Except):
cause = Except(UNEXPECTED, unicode(cause), trace=_extract_traceback(0))
trace = extract_stack(1)
e = Except(UNEXPECTED, template, params, cause, trace)
Log.note(
"{{error}}",
error=e,
log_context=set_default({"context": WARNING}, log_context),
stack_depth=stack_depth + 1
)
def error(
cls,
template, # human readable template
default_params={}, # parameters for template
cause=None, # pausible cause
stack_depth=0,
**more_params
):
"""
raise an exception with a trace for the cause too
"""
if default_params and isinstance(listwrap(default_params)[0], BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
add_to_trace = False
cause = unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)])
trace = extract_stack(stack_depth + 1)
if add_to_trace:
cause[0].trace.extend(trace[1:])
e = Except(ERROR, template, params, cause, trace)
raise e
def format_cube(T, select, query=None):
table = format_table(T, select, query)
if len(table.data) == 0:
return Cube(
select,
edges=[{
"name": "rownum",
"domain": {
"type": "rownum",
"min": 0,
"max": 0,
"interval": 1
}
}],
data={h: Matrix(list=[])
for i, h in enumerate(table.header)})
cols = zip(*unwrap(table.data))
return Cube(
select,
edges=[{
"name": "rownum",
"domain": {
"type": "rownum",
"min": 0,
"max": len(table.data),
"interval": 1
}
}],
data={h: Matrix(list=cols[i])
for i, h in enumerate(table.header)})
def post(sql):
# FIND OUT THE default DOMAIN SIZES
result = self.db.column_query(sql)
num_edges = len(edges)
for e, edge in enumerate(edges):
domain = edge.domain
if domain.type == "default":
domain.type = "set"
parts = set(result[e])
domain.partitions = [{
"index": i,
"value": p
} for i, p in enumerate(parts)]
domain.map = {p: i for i, p in enumerate(parts)}
else:
Log.error("Do not know what to do here, yet")
# FILL THE DATA CUBE
maps = [(unwrap(e.domain.map), result[i])
for i, e in enumerate(edges)]
cubes = DictList()
for c, s in enumerate(select):
data = Matrix(*[
len(e.domain.partitions) + (1 if e.allow_nulls else 0)
for e in edges
])
for rownum, value in enumerate(result[c + num_edges]):
coord = [m[r[rownum]] for m, r in maps]
data[coord] = value
cubes.append(data)
if isinstance(query.select, list):
return cubes
else:
return cubes[0]
def tuple(data, field_name):
"""
RETURN LIST OF TUPLES
"""
if isinstance(data, Cube):
Log.error("not supported yet")
if isinstance(data, FlatList):
Log.error("not supported yet")
if isinstance(field_name, Mapping) and "value" in field_name:
# SIMPLIFY {"value":value} AS STRING
field_name = field_name["value"]
# SIMPLE PYTHON ITERABLE ASSUMED
if isinstance(field_name, basestring):
if len(split_field(field_name)) == 1:
return [(d[field_name], ) for d in data]
else:
path = split_field(field_name)
output = []
flat_list._tuple1(data, path, 0, output)
return output
elif isinstance(field_name, list):
paths = [_select_a_field(f) for f in field_name]
output = DictList()
_tuple((), unwrap(data), paths, 0, output)
return output
else:
paths = [_select_a_field(field_name)]
output = DictList()
_tuple((), data, paths, 0, output)
return output
def unexpected(
cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params
):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
if cause and not isinstance(cause, Except):
cause = Except(exceptions.UNEXPECTED, unicode(cause), trace=exceptions._extract_traceback(0))
trace = exceptions.extract_stack(1)
e = Except(exceptions.UNEXPECTED, template, params, cause, trace)
Log.note(
"{{error}}",
error=e,
log_context=set_default({"context": exceptions.WARNING}, log_context),
stack_depth=stack_depth + 1
)
def note(cls, template, default_params={}, stack_depth=0, **more_params):
if len(template) > 10000:
template = template[:10000]
params = dict(unwrap(default_params), **more_params)
log_params = Dict(
template=template,
params=params,
timestamp=datetime.utcnow(),
)
if not template.startswith("\n") and template.find("\n") > -1:
template = "\n" + template
if cls.trace:
log_template = "{{timestamp|datetime}} - {{thread.name}} - {{location.file}}:{{location.line}} ({{location.method}}) - " + template.replace("{{", "{{params.")
f = sys._getframe(stack_depth + 1)
log_params.location = {
"line": f.f_lineno,
"file": f.f_code.co_filename.split(os.sep)[-1],
"method": f.f_code.co_name
}
thread = Thread.current()
log_params.thread = {"name": thread.name, "id": thread.id}
else:
log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.")
cls.main_log.write(log_template, log_params)
def warning(
cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params
):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
if "values" in more_params.keys():
Log.error("Can not handle a logging parameter by name `values`")
params = dict(unwrap(default_params), **more_params)
cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
trace = exceptions.extract_stack(stack_depth + 1)
e = Except(exceptions.WARNING, template, params, cause, trace)
Log.note(
"{{error|unicode}}",
error=e,
log_context=set_default({"context": exceptions.WARNING}, log_context),
stack_depth=stack_depth + 1
)
def error(
cls,
template, # human readable template
default_params={}, # parameters for template
cause=None, # pausible cause
stack_depth=0,
**more_params
):
"""
raise an exception with a trace for the cause too
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if default_params and isinstance(listwrap(default_params)[0], BaseException):
cause = default_params
default_params = {}
params = dict(unwrap(default_params), **more_params)
add_to_trace = False
cause = wrap(unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)]))
trace = exceptions.extract_stack(stack_depth + 1)
if add_to_trace:
cause[0].trace.extend(trace[1:])
e = Except(exceptions.ERROR, template, params, cause, trace)
raise e
def __setitem__(self, key, value):
if key == "":
from pyLibrary.debugs.logs import Log
Log.error("key is empty string. Probably a bad idea")
if isinstance(key, str):
key = key.decode("utf8")
try:
d = _get(self, "_dict")
value = unwrap(value)
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
return self
seq = split_field(key)
for k in seq[:-1]:
d = _getdefault(d, k)
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return self
except Exception, e:
raise e
def post(sql):
# FIND OUT THE default DOMAIN SIZES
result = self.db.column_query(sql)
num_edges = len(edges)
for e, edge in enumerate(edges):
domain = edge.domain
if domain.type == "default":
domain.type = "set"
parts = set(result[e])
domain.partitions = [{"index": i, "value": p} for i, p in enumerate(parts)]
domain.map = {p: i for i, p in enumerate(parts)}
else:
Log.error("Do not know what to do here, yet")
# FILL THE DATA CUBE
maps = [(unwrap(e.domain.map), result[i]) for i, e in enumerate(edges)]
cubes = DictList()
for c, s in enumerate(select):
data = Matrix(*[len(e.domain.partitions) + (1 if e.allow_nulls else 0) for e in edges])
for rownum, value in enumerate(result[c + num_edges]):
coord = [m[r[rownum]] for m, r in maps]
data[coord] = value
cubes.append(data)
if isinstance(query.select, list):
return cubes
else:
return cubes[0]
def select(self, key):
"""
simple `select`
"""
if not dictwrap:
_late_import()
return DictList(vals=[unwrap(dictwrap(v)[key]) for v in _get(self, "list")])
def get(self, key):
"""
simple `select`
"""
if not _dictwrap:
_late_import()
return DictList(vals=[unwrap(coalesce(_dictwrap(v), Null)[key]) for v in _get(self, "list")])
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}",
ref=ref,
new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def call(self, proc_name, params):
self._execute_backlog()
params = [unwrap(v) for v in params]
try:
self.cursor.callproc(proc_name, params)
self.cursor.close()
self.cursor = self.db.cursor()
except Exception, e:
Log.error("Problem calling procedure " + proc_name, e)
def call(self, proc_name, params):
self._execute_backlog()
params = [unwrap(v) for v in params]
try:
self.cursor.callproc(proc_name, params)
self.cursor.close()
self.cursor = self.db.cursor()
except Exception, e:
Log.error("Problem calling procedure " + proc_name, e)
def es_fieldop(es, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
FromES.query = {
"filtered": {
"query": {
"match_all": {}
},
"filter": simplify_esfilter(query.where)
}
}
FromES.size = coalesce(query.limit, 200000)
FromES.fields = DictList()
for s in select.value:
if s == "*":
FromES.fields = None
elif isinstance(s, list):
FromES.fields.extend(s)
elif isinstance(s, Mapping):
FromES.fields.extend(s.values())
else:
FromES.fields.append(s)
FromES.sort = [{s.field: "asc" if s.sort >= 0 else "desc"} for s in query.sort]
data = es09.util.post(es, FromES, query.limit)
T = data.hits.hits
matricies = {}
for s in select:
if s.value == "*":
matricies[s.name] = Matrix.wrap([t._source for t in T])
elif isinstance(s.value, Mapping):
# for k, v in s.value.items():
# matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T])
matricies[s.name] = Matrix.wrap([{k: unwrap(t.fields).get(v, None) for k, v in s.value.items()}for t in T])
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([tuple(unwrap(t.fields).get(ss, None) for ss in s.value) for t in T])
elif not s.value:
matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T])
else:
try:
matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T])
except Exception, e:
Log.error("", e)
def _replace_ref(node, url):
if url.path.endswith("/"):
url.path = url.path[:-1]
if isinstance(node, Mapping):
ref = None
output = {}
for k, v in node.items():
if k == "$ref":
ref = URL(v)
else:
output[k] = _replace_ref(v, url)
if not ref:
return output
node = output
if not ref.scheme and not ref.path:
# DO NOT TOUCH LOCAL REF YET
output["$ref"] = ref
return output
if not ref.scheme:
# SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH
# REQUIRES THE CURRENT DOCUMENT'S SCHEME
ref.scheme = url.scheme
# FIND THE SCHEME AND LOAD IT
if ref.scheme in scheme_loaders:
new_value = scheme_loaders[ref.scheme](ref, url)
else:
raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme)
if ref.fragment:
new_value = dot.get_attr(new_value, ref.fragment)
if DEBUG:
_Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value)
if not output:
output = new_value
else:
output = unwrap(set_default(output, new_value))
if DEBUG:
_Log.note("Return {{output}}", output=output)
return output
elif isinstance(node, list):
output = [_replace_ref(n, url) for n in node]
# if all(p[0] is p[1] for p in zip(output, node)):
# return node
return output
return node
def reverse(vals):
# TODO: Test how to do this fastest
l = len(vals)
output = [None] * l
for v in unwrap(vals):
l -= 1
output[l] = v
return wrap(output)
def argparse(defs):
parser = _argparse.ArgumentParser()
for d in listwrap(defs):
args = d.copy()
name = args.name
args.name = None
parser.add_argument(*unwrap(listwrap(name)), **args)
namespace = parser.parse_args()
output = {k: getattr(namespace, k) for k in vars(namespace)}
return wrap(output)
def reverse(vals):
# TODO: Test how to do this fastest
l = len(vals)
output = [None] * l
for v in unwrap(vals):
l -= 1
output[l] = v
return wrap(output)
def __init__(self, name, data, schema=None):
#TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(unwrap(data))
Container.__init__(self, data, schema)
if schema == None:
self._schema = get_schema_from_list(data)
else:
self._schema = schema
self.name = name
self.data = data
self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def params_pack(params, *args):
settings = {}
for a in args:
for k, v in a.items():
k = unicode(k)
if k in settings:
continue
settings[k] = v
output = {str(k): unwrap(settings[k]) for k in params if k in settings}
return output
def __init__(self, *args, **kwargs):
"""
CALLING Dict(**something) WILL RESULT IN A COPY OF something, WHICH
IS UNLIKELY TO BE USEFUL. USE wrap() INSTEAD
"""
if DEBUG:
d = _get(self, "_dict")
for k, v in kwargs.items():
d[literal_field(k)] = unwrap(v)
else:
if args:
args0 = args[0]
if isinstance(args0, Mapping):
_set(self, "_dict", args0)
else:
_set(self, "_dict", _get(args[0], "__dict__"))
elif kwargs:
_set(self, "_dict", unwrap(kwargs))
else:
_set(self, "_dict", {})
def __init__(self, *args, **kwargs):
"""
CALLING Dict(**something) WILL RESULT IN A COPY OF something, WHICH
IS UNLIKELY TO BE USEFUL. USE wrap() INSTEAD
"""
if DEBUG:
d = _get(self, "_dict")
for k, v in kwargs.items():
d[literal_field(k)] = unwrap(v)
else:
if args:
args0 = args[0]
if isinstance(args0, Mapping):
_set(self, "_dict", args0)
else:
_set(self, "_dict", _get(args[0], "__dict__"))
elif kwargs:
_set(self, "_dict", unwrap(kwargs))
else:
_set(self, "_dict", {})
def params_pack(params, *args):
settings = {}
for a in args:
for k, v in a.items():
k = unicode(k)
if k in settings:
continue
settings[k] = v
output = {str(k): unwrap(settings[k]) for k in params if k in settings}
return output
def __init__(self, name, data, schema=None):
#TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(unwrap(data))
Container.__init__(self, data, schema)
if schema == None:
self._schema = get_schema_from_list(data)
else:
self._schema = schema
self.name = name
self.data = data
self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def get(self, key):
"""
simple `select`
"""
if not _dictwrap:
_late_import()
return DictList(vals=[
unwrap(coalesce(_dictwrap(v), Null)[key])
for v in _get(self, "list")
])
def select(data, field_name):
"""
return list with values from field_name
"""
if isinstance(data, Cube):
return data._select(_normalize_selects(field_name))
if isinstance(data, FlatList):
return data.select(field_name)
if isinstance(data, UniqueIndex):
data = data._data.values(
) # THE SELECT ROUTINE REQUIRES dicts, NOT Dict WHILE ITERATING
if isinstance(data, Mapping):
return select_one(data, field_name)
if isinstance(field_name, Mapping):
field_name = wrap(field_name)
if field_name.value in ["*", "."]:
return data
if field_name.value:
# SIMPLIFY {"value":value} AS STRING
field_name = field_name.value
# SIMPLE PYTHON ITERABLE ASSUMED
if isinstance(field_name, basestring):
path = split_field(field_name)
if len(path) == 1:
return DictList([d[field_name] for d in data])
else:
output = DictList()
flat_list._select1(data, path, 0, output)
return output
elif isinstance(field_name, list):
keys = [_select_a_field(wrap(f)) for f in field_name]
return _select(Dict(), unwrap(data), keys, 0)
else:
keys = [_select_a_field(field_name)]
return _select(Dict(), unwrap(data), keys, 0)
def es_aggop(es, mvel, query):
select = listwrap(query.select)
FromES = build_es_query(query)
isSimple = AND(aggregates[s.aggregate] == "count" for s in select)
if isSimple:
return es_countop(es, query) # SIMPLE, USE TERMS FACET INSTEAD
value2facet = dict() # ONLY ONE FACET NEEDED PER
name2facet = dict() # MAP name TO FACET WITH STATS
for s in select:
if s.value not in value2facet:
if is_keyword(s.value):
unwrap(FromES.facets)[s.name] = {
"statistical": {
"field": s.value
},
"facet_filter": simplify_esfilter(query.where)
}
else:
unwrap(FromES.facets)[s.name] = {
"statistical": {
"script":
es09.expressions.compile_expression(s.value, query)
},
"facet_filter": simplify_esfilter(query.where)
}
value2facet[s.value] = s.name
name2facet[s.name] = value2facet[s.value]
data = es09.util.post(es, FromES, query.limit)
matricies = {
s.name: Matrix(value=fix_es_stats(data.facets[literal_field(s.name)])[
aggregates[s.aggregate]])
for s in select
}
cube = Cube(query.select, [], matricies)
cube.frum = query
return cube
def __setitem__(self, i, y):
try:
_list = _get(self, "list")
if i <= len(_list):
for i in range(len(_list), i):
_list.append(None)
_list[i] = unwrap(y)
except Exception, e:
if not _dictwrap:
_late_import()
_Log.error("problem", cause = e)
def __setitem__(self, i, y):
try:
_list = _get(self, "list")
if i <= len(_list):
for i in range(len(_list), i):
_list.append(None)
_list[i] = unwrap(y)
except Exception, e:
if not _dictwrap:
_late_import()
_Log.error("problem", cause=e)
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
aws_args = dict(
region_name=settings.aws.region,
aws_access_key_id=unwrap(settings.aws.aws_access_key_id),
aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key)
)
ec2_conn = boto_ec2.connect_to_region(**aws_args)
instances = _get_managed_instances(ec2_conn, settings.name)
for i in instances:
Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", insance_id=i.id, name=i.tags["Name"], ip=i.ip_address)
_config_fabric(settings.fabric, i)
_refresh_indexer()
except Exception, e:
Log.error("Problem with etl", e)
def select(data, field_name):
"""
return list with values from field_name
"""
if isinstance(data, Cube):
return data._select(_normalize_selects(field_name))
if isinstance(data, FlatList):
return data.select(field_name)
if isinstance(data, UniqueIndex):
data = data._data.values() # THE SELECT ROUTINE REQUIRES dicts, NOT Dict WHILE ITERATING
if isinstance(data, Mapping):
return select_one(data, field_name)
if isinstance(field_name, Mapping):
field_name = wrap(field_name)
if field_name.value in ["*", "."]:
return data
if field_name.value:
# SIMPLIFY {"value":value} AS STRING
field_name = field_name.value
# SIMPLE PYTHON ITERABLE ASSUMED
if isinstance(field_name, basestring):
path = split_field(field_name)
if len(path) == 1:
return DictList([d[field_name] for d in data])
else:
output = DictList()
flat_list._select1(data, path, 0, output)
return output
elif isinstance(field_name, list):
keys = [_select_a_field(wrap(f)) for f in field_name]
return _select(Dict(), unwrap(data), keys, 0)
else:
keys = [_select_a_field(field_name)]
return _select(Dict(), unwrap(data), keys, 0)
def filter(data, where):
"""
where - a function that accepts (record, rownum, rows) and returns boolean
"""
if len(data) == 0 or where == None or where == TRUE_FILTER:
return data
if isinstance(data, Container):
return data.filter(where)
if isinstance(data, (list, set)):
temp = jx_expression_to_function(where)
dd = wrap(data)
return wrap([unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)])
else:
Log.error("Do not know how to handle type {{type}}", type=data.__class__.__name__)
try:
return drill_filter(where, data)
except Exception, _:
# WOW! THIS IS INEFFICIENT!
return wrap([unwrap(d) for d in drill_filter(where, [DictObject(d) for d in data])])
def __init__(
self,
aws_access_key_id=None, # CREDENTIAL
aws_secret_access_key=None, # CREDENTIAL
region=None, # NAME OF AWS REGION, REQUIRED FOR SOME BUCKETS
settings=None):
self.settings = settings
try:
if not settings.region:
self.connection = boto.connect_s3(
aws_access_key_id=unwrap(self.settings.aws_access_key_id),
aws_secret_access_key=unwrap(
self.settings.aws_secret_access_key))
else:
self.connection = boto.s3.connect_to_region(
self.settings.region,
aws_access_key_id=unwrap(self.settings.aws_access_key_id),
aws_secret_access_key=unwrap(
self.settings.aws_secret_access_key))
except Exception, e:
Log.error("Problem connecting to S3", e)
def _send_email(self):
try:
if self.accumulation:
conn = connect_to_region(
self.settings.region,
aws_access_key_id=unwrap(self.settings.aws_access_key_id),
aws_secret_access_key=unwrap(
self.settings.aws_secret_access_key))
conn.send_email(source=self.settings.from_address,
to_addresses=listwrap(
self.settings.to_address),
subject=self.settings.subject,
body="\n\n".join(self.accumulation),
format="text")
conn.close()
self.next_send = Date.now() + WAIT_TO_SEND_MORE
self.accumulation = []
except Exception, e:
self.next_send = Date.now() + WAIT_TO_SEND_MORE
Log.warning("Could not send", e)
def __setattr__(self, key, value):
if isinstance(key, str):
ukey = key.decode("utf8")
else:
ukey = key
d = self
value = unwrap(value)
if value is None:
dict.pop(d, key, None)
else:
dict.__setitem__(d, ukey, value)
return self
def __setattr__(self, key, value):
if isinstance(key, str):
ukey = key.decode("utf8")
else:
ukey = key
d = _get(self, "_dict")
value = unwrap(value)
if value is None:
d = _get(self, "_dict")
d.pop(key, None)
else:
d[ukey] = value
return self
