def __init__(self, name, region, aws_access_key_id=None, aws_secret_access_key=None, debug=False, settings=None): self.settings = settings self.pending = [] if settings.region not in [r.name for r in sqs.regions()]: Log.error("Can not find region {{region}} in {{regions}}", region=settings.region, regions=[r.name for r in sqs.regions()]) conn = sqs.connect_to_region( region_name=unwrap(settings.region), aws_access_key_id=unwrap(settings.aws_access_key_id), aws_secret_access_key=unwrap(settings.aws_secret_access_key), ) self.queue = conn.get_queue(settings.name) if self.queue == None: Log.error( "Can not find queue with name {{queue}} in region {{region}}", queue=settings.name, region=settings.region)
def __setitem__(self, key, value): if key == "": from pyLibrary.debugs.logs import Log Log.error("key is empty string. Probably a bad idea") if key == ".": # SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping; # HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS unwrap() v = unwrap(value) _set(self, "_dict", v) return v if isinstance(key, str): key = key.decode("utf8") try: d = _get(self, "_dict") value = unwrap(value) if key.find(".") == -1: if value is None: d.pop(key, None) else: d[key] = value return self seq = _split_field(key) for k in seq[:-1]: d = _getdefault(d, k) if value == None: d.pop(seq[-1], None) else: d[seq[-1]] = value return self except Exception, e: raise e
def main(): try: settings = startup.read_settings() constants.set(settings.constants) Log.start(settings.debug) aws_args = dict( region_name=settings.aws.region, aws_access_key_id=unwrap(settings.aws.aws_access_key_id), aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key) ) ec2_conn = boto_ec2.connect_to_region(**aws_args) instances = _get_managed_instances(ec2_conn, settings.name) for i in instances: Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", insance_id=i.id, name=i.tags["Name"], ip=i.ip_address) _config_fabric(settings.fabric, i) try: _refresh_etl() # TODO: UPON FAILURE, TERMINATE INSTANCE AND SPOT REQUEST except Exception, e: ec2_conn.terminate_instances([i.id]) Log.warning("Problem resetting {{instance}}, terminated", instance=i.id, cause=e) except Exception, e: Log.error("Problem with etl", e)
def filter(data, where): """ where - a function that accepts (record, rownum, rows) and returns boolean """ if len(data) == 0 or where == None or where == TRUE_FILTER: return data if isinstance(data, Container): return data.filter(where) if isinstance(data, (list, set)): temp = jx_expression_to_function(where) dd = wrap(data) return wrap( [unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)]) else: Log.error("Do not know how to handle type {{type}}", type=data.__class__.__name__) try: return drill_filter(where, data) except Exception, _: # WOW! THIS IS INEFFICIENT! return wrap([ unwrap(d) for d in drill_filter(where, [DictObject(d) for d in data]) ])
def es_fieldop(es, query): FromES = es09.util.build_es_query(query) select = listwrap(query.select) FromES.query = { "filtered": { "query": { "match_all": {} }, "filter": simplify_esfilter(jx_expression(query.where).to_esfilter()) } } FromES.size = coalesce(query.limit, 200000) FromES.fields = DictList() for s in select.value: if s == "*": FromES.fields = None elif isinstance(s, list): FromES.fields.extend(s) elif isinstance(s, Mapping): FromES.fields.extend(s.values()) else: FromES.fields.append(s) FromES.sort = [{ s.field: "asc" if s.sort >= 0 else "desc" } for s in query.sort] data = es09.util.post(es, FromES, query.limit) T = data.hits.hits matricies = {} for s in select: if s.value == "*": matricies[s.name] = Matrix.wrap([t._source for t in T]) elif isinstance(s.value, Mapping): # for k, v in s.value.items(): # matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T]) matricies[s.name] = Matrix.wrap([{ k: unwrap(t.fields).get(v, None) for k, v in s.value.items() } for t in T]) elif isinstance(s.value, list): matricies[s.name] = Matrix.wrap([ tuple(unwrap(t.fields).get(ss, None) for ss in s.value) for t in T ]) elif not s.value: matricies[s.name] = Matrix.wrap( [unwrap(t.fields).get(s.value, None) for t in T]) else: try: matricies[s.name] = Matrix.wrap( [unwrap(t.fields).get(s.value, None) for t in T]) except Exception, e: Log.error("", e)
def extend(self, records): """ JUST SO WE MODEL A Queue """ records = {v["id"]: v["value"] for v in records} unwrap(self.data).update(records) data_as_json = convert.value2json(self.data, pretty=True) File(self.filename).write(data_as_json) Log.note("{{num}} documents added", num= len(records))
def extend(self, records): """ JUST SO WE MODEL A Queue """ records = {v["id"]: v["value"] for v in records} unwrap(self.data).update(records) data_as_json = convert.value2json(self.data, pretty=True) File(self.filename).write(data_as_json) Log.note("{{num}} documents added", num=len(records))
def warning(cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if isinstance(default_params, BaseException): cause = default_params default_params = {} if "values" in more_params.keys(): Log.error("Can not handle a logging parameter by name `values`") params = dict(unwrap(default_params), **more_params) cause = unwraplist([Except.wrap(c) for c in listwrap(cause)]) trace = exceptions.extract_stack(stack_depth + 1) e = Except(exceptions.WARNING, template, params, cause, trace) Log.note("{{error|unicode}}", error=e, log_context=set_default({"context": exceptions.WARNING}, log_context), stack_depth=stack_depth + 1)
def dict2Multiset(dic): if dic == None: return None output = Multiset() output.dic = unwrap(dic).copy() return output
def __setitem__(self, key, value): if key == "": from pyLibrary.debugs.logs import Log Log.error("key is empty string. Probably a bad idea") if isinstance(key, str): key = key.decode("utf8") d=self try: value = unwrap(value) if key.find(".") == -1: if value is None: dict.pop(d, key, None) else: dict.__setitem__(d, key, value) return self seq = _split_field(key) for k in seq[:-1]: d = _getdefault(d, k) if value == None: dict.pop(d, seq[-1], None) else: dict.__setitem__(d, seq[-1], value) return self except Exception, e: raise e
def error( cls, template, # human readable template default_params={}, # parameters for template cause=None, # pausible cause stack_depth=0, **more_params ): """ raise an exception with a trace for the cause too :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if default_params and isinstance(listwrap(default_params)[0], BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) add_to_trace = False cause = wrap(unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)])) trace = exceptions.extract_stack(stack_depth + 1) if add_to_trace: cause[0].trace.extend(trace[1:]) e = Except(exceptions.ERROR, template, params, cause, trace) raise e
def warning( cls, template, default_params={}, cause=None, stack_depth=0, # stack trace offset (==1 if you do not want to report self) **more_params ): if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) cause = unwraplist([Except.wrap(c) for c in listwrap(cause)]) trace = extract_stack(stack_depth + 1) e = Except(WARNING, template, params, cause, trace) Log.note( unicode(e), { "warning": {# REDUNDANT INFO "template": template, "params": params, "cause": cause, "trace": trace } }, stack_depth=stack_depth + 1 )
def test_request(self): # MAKE SOME DATA data = { "constant": "this is a test", "random-data": convert.bytes2base64(Random.bytes(100)) } client = Client(settings.url, unwrap(settings.hawk)) # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21 link, id = client.send(data) Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id) # FILL THE REST OF THE FILE Log.note("Add ing {{num}} more...", num=99-id) for i in range(id + 1, storage.BATCH_SIZE): l, k = client.send(data) if l != link: Log.error("Expecting rest of data to have same link") # TEST LINK HAS DATA raw_content = requests.get(link).content content = convert.zip2bytes(raw_content) for line in convert.utf82unicode(content).split("\n"): data = convert.json2value(line) if data.etl.id == id: Log.note("Data {{id}} found", id=id) break else: Log.error("Expecting to find data at link")
def note(cls, template, default_params={}, stack_depth=0, **more_params): if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = Dict(template=template, params=params, timestamp=datetime.utcnow(), machine=machine_metadata.name) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{machine}} - {{timestamp|datetime}} - {{thread.name}} - {{location.file}}:{{location.line}} ({{location.method}}) - " + template.replace( "{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": f.f_code.co_filename.split(os.sep)[-1], "method": f.f_code.co_name } thread = Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace( "{{", "{{params.") cls.main_log.write(log_template, log_params)
def unexpected(cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) if cause and not isinstance(cause, Except): cause = Except(exceptions.UNEXPECTED, unicode(cause), trace=exceptions._extract_traceback(0)) trace = exceptions.extract_stack(1) e = Except(exceptions.UNEXPECTED, template, params, cause, trace) Log.note("{{error}}", error=e, log_context=set_default({"context": exceptions.WARNING}, log_context), stack_depth=stack_depth + 1)
def warning( cls, template, default_params={}, cause=None, stack_depth=0, # stack trace offset (==1 if you do not want to report self) **more_params): if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) cause = unwraplist([Except.wrap(c) for c in listwrap(cause)]) trace = extract_stack(stack_depth + 1) e = Except(WARNING, template, params, cause, trace) Log.note( unicode(e), { "warning": { # REDUNDANT INFO "template": template, "params": params, "cause": cause, "trace": trace } }, stack_depth=stack_depth + 1)
def unexpected(cls, template, default_params={}, cause=None, **more_params): if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) if cause and not isinstance(cause, Except): cause = Except(UNEXPECTED, unicode(cause), trace=extract_tb(0)) trace = extract_stack(1) e = Except(UNEXPECTED, template, params, cause, trace) Log.note( unicode(e), { "warning": { "template": template, "params": params, "cause": cause, "trace": trace } })
def tuple(data, field_name): """ RETURN LIST OF TUPLES """ if isinstance(data, Cube): Log.error("not supported yet") if isinstance(data, FlatList): Log.error("not supported yet") if isinstance(field_name, Mapping) and "value" in field_name: # SIMPLIFY {"value":value} AS STRING field_name = field_name["value"] # SIMPLE PYTHON ITERABLE ASSUMED if isinstance(field_name, basestring): if len(split_field(field_name)) == 1: return [(d[field_name],) for d in data] else: path = split_field(field_name) output = [] flat_list._tuple1(data, path, 0, output) return output elif isinstance(field_name, list): paths = [_select_a_field(f) for f in field_name] output = DictList() _tuple((), unwrap(data), paths, 0, output) return output else: paths = [_select_a_field(field_name)] output = DictList() _tuple((), data, paths, 0, output) return output
def unexpected( cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params ): if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) if cause and not isinstance(cause, Except): cause = Except(UNEXPECTED, unicode(cause), trace=_extract_traceback(0)) trace = extract_stack(1) e = Except(UNEXPECTED, template, params, cause, trace) Log.note( "{{error}}", error=e, log_context=set_default({"context": WARNING}, log_context), stack_depth=stack_depth + 1 )
def error( cls, template, # human readable template default_params={}, # parameters for template cause=None, # pausible cause stack_depth=0, **more_params ): """ raise an exception with a trace for the cause too """ if default_params and isinstance(listwrap(default_params)[0], BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) add_to_trace = False cause = unwraplist([Except.wrap(c, stack_depth=1) for c in listwrap(cause)]) trace = extract_stack(stack_depth + 1) if add_to_trace: cause[0].trace.extend(trace[1:]) e = Except(ERROR, template, params, cause, trace) raise e
def format_cube(T, select, query=None): table = format_table(T, select, query) if len(table.data) == 0: return Cube( select, edges=[{ "name": "rownum", "domain": { "type": "rownum", "min": 0, "max": 0, "interval": 1 } }], data={h: Matrix(list=[]) for i, h in enumerate(table.header)}) cols = zip(*unwrap(table.data)) return Cube( select, edges=[{ "name": "rownum", "domain": { "type": "rownum", "min": 0, "max": len(table.data), "interval": 1 } }], data={h: Matrix(list=cols[i]) for i, h in enumerate(table.header)})
def post(sql): # FIND OUT THE default DOMAIN SIZES result = self.db.column_query(sql) num_edges = len(edges) for e, edge in enumerate(edges): domain = edge.domain if domain.type == "default": domain.type = "set" parts = set(result[e]) domain.partitions = [{ "index": i, "value": p } for i, p in enumerate(parts)] domain.map = {p: i for i, p in enumerate(parts)} else: Log.error("Do not know what to do here, yet") # FILL THE DATA CUBE maps = [(unwrap(e.domain.map), result[i]) for i, e in enumerate(edges)] cubes = DictList() for c, s in enumerate(select): data = Matrix(*[ len(e.domain.partitions) + (1 if e.allow_nulls else 0) for e in edges ]) for rownum, value in enumerate(result[c + num_edges]): coord = [m[r[rownum]] for m, r in maps] data[coord] = value cubes.append(data) if isinstance(query.select, list): return cubes else: return cubes[0]
def tuple(data, field_name): """ RETURN LIST OF TUPLES """ if isinstance(data, Cube): Log.error("not supported yet") if isinstance(data, FlatList): Log.error("not supported yet") if isinstance(field_name, Mapping) and "value" in field_name: # SIMPLIFY {"value":value} AS STRING field_name = field_name["value"] # SIMPLE PYTHON ITERABLE ASSUMED if isinstance(field_name, basestring): if len(split_field(field_name)) == 1: return [(d[field_name], ) for d in data] else: path = split_field(field_name) output = [] flat_list._tuple1(data, path, 0, output) return output elif isinstance(field_name, list): paths = [_select_a_field(f) for f in field_name] output = DictList() _tuple((), unwrap(data), paths, 0, output) return output else: paths = [_select_a_field(field_name)] output = DictList() _tuple((), data, paths, 0, output) return output
def unexpected( cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params ): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if isinstance(default_params, BaseException): cause = default_params default_params = {} params = dict(unwrap(default_params), **more_params) if cause and not isinstance(cause, Except): cause = Except(exceptions.UNEXPECTED, unicode(cause), trace=exceptions._extract_traceback(0)) trace = exceptions.extract_stack(1) e = Except(exceptions.UNEXPECTED, template, params, cause, trace) Log.note( "{{error}}", error=e, log_context=set_default({"context": exceptions.WARNING}, log_context), stack_depth=stack_depth + 1 )
def note(cls, template, default_params={}, stack_depth=0, **more_params): if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = Dict( template=template, params=params, timestamp=datetime.utcnow(), ) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{timestamp|datetime}} - {{thread.name}} - {{location.file}}:{{location.line}} ({{location.method}}) - " + template.replace("{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": f.f_code.co_filename.split(os.sep)[-1], "method": f.f_code.co_name } thread = Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.") cls.main_log.write(log_template, log_params)
def warning( cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params ): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if isinstance(default_params, BaseException): cause = default_params default_params = {} if "values" in more_params.keys(): Log.error("Can not handle a logging parameter by name `values`") params = dict(unwrap(default_params), **more_params) cause = unwraplist([Except.wrap(c) for c in listwrap(cause)]) trace = exceptions.extract_stack(stack_depth + 1) e = Except(exceptions.WARNING, template, params, cause, trace) Log.note( "{{error|unicode}}", error=e, log_context=set_default({"context": exceptions.WARNING}, log_context), stack_depth=stack_depth + 1 )
def __setitem__(self, key, value): if key == "": from pyLibrary.debugs.logs import Log Log.error("key is empty string. Probably a bad idea") if isinstance(key, str): key = key.decode("utf8") try: d = _get(self, "_dict") value = unwrap(value) if key.find(".") == -1: if value is None: d.pop(key, None) else: d[key] = value return self seq = split_field(key) for k in seq[:-1]: d = _getdefault(d, k) if value == None: d.pop(seq[-1], None) else: d[seq[-1]] = value return self except Exception, e: raise e
def post(sql): # FIND OUT THE default DOMAIN SIZES result = self.db.column_query(sql) num_edges = len(edges) for e, edge in enumerate(edges): domain = edge.domain if domain.type == "default": domain.type = "set" parts = set(result[e]) domain.partitions = [{"index": i, "value": p} for i, p in enumerate(parts)] domain.map = {p: i for i, p in enumerate(parts)} else: Log.error("Do not know what to do here, yet") # FILL THE DATA CUBE maps = [(unwrap(e.domain.map), result[i]) for i, e in enumerate(edges)] cubes = DictList() for c, s in enumerate(select): data = Matrix(*[len(e.domain.partitions) + (1 if e.allow_nulls else 0) for e in edges]) for rownum, value in enumerate(result[c + num_edges]): coord = [m[r[rownum]] for m, r in maps] data[coord] = value cubes.append(data) if isinstance(query.select, list): return cubes else: return cubes[0]
def select(self, key): """ simple `select` """ if not dictwrap: _late_import() return DictList(vals=[unwrap(dictwrap(v)[key]) for v in _get(self, "list")])
def get(self, key): """ simple `select` """ if not _dictwrap: _late_import() return DictList(vals=[unwrap(coalesce(_dictwrap(v), Null)[key]) for v in _get(self, "list")])
def _replace_ref(node, url): if url.path.endswith("/"): url.path = url.path[:-1] if isinstance(node, Mapping): ref = None output = {} for k, v in node.items(): if k == "$ref": ref = URL(v) else: output[k] = _replace_ref(v, url) if not ref: return output node = output if not ref.scheme and not ref.path: # DO NOT TOUCH LOCAL REF YET output["$ref"] = ref return output if not ref.scheme: # SCHEME RELATIVE IMPLIES SAME PROTOCOL AS LAST TIME, WHICH # REQUIRES THE CURRENT DOCUMENT'S SCHEME ref.scheme = url.scheme # FIND THE SCHEME AND LOAD IT if ref.scheme in scheme_loaders: new_value = scheme_loaders[ref.scheme](ref, url) else: raise _Log.error("unknown protocol {{scheme}}", scheme=ref.scheme) if ref.fragment: new_value = dot.get_attr(new_value, ref.fragment) if DEBUG: _Log.note("Replace {{ref}} with {{new_value}}", ref=ref, new_value=new_value) if not output: output = new_value else: output = unwrap(set_default(output, new_value)) if DEBUG: _Log.note("Return {{output}}", output=output) return output elif isinstance(node, list): output = [_replace_ref(n, url) for n in node] # if all(p[0] is p[1] for p in zip(output, node)): # return node return output return node
def call(self, proc_name, params): self._execute_backlog() params = [unwrap(v) for v in params] try: self.cursor.callproc(proc_name, params) self.cursor.close() self.cursor = self.db.cursor() except Exception, e: Log.error("Problem calling procedure " + proc_name, e)
def es_fieldop(es, query): FromES = es09.util.build_es_query(query) select = listwrap(query.select) FromES.query = { "filtered": { "query": { "match_all": {} }, "filter": simplify_esfilter(query.where) } } FromES.size = coalesce(query.limit, 200000) FromES.fields = DictList() for s in select.value: if s == "*": FromES.fields = None elif isinstance(s, list): FromES.fields.extend(s) elif isinstance(s, Mapping): FromES.fields.extend(s.values()) else: FromES.fields.append(s) FromES.sort = [{s.field: "asc" if s.sort >= 0 else "desc"} for s in query.sort] data = es09.util.post(es, FromES, query.limit) T = data.hits.hits matricies = {} for s in select: if s.value == "*": matricies[s.name] = Matrix.wrap([t._source for t in T]) elif isinstance(s.value, Mapping): # for k, v in s.value.items(): # matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T]) matricies[s.name] = Matrix.wrap([{k: unwrap(t.fields).get(v, None) for k, v in s.value.items()}for t in T]) elif isinstance(s.value, list): matricies[s.name] = Matrix.wrap([tuple(unwrap(t.fields).get(ss, None) for ss in s.value) for t in T]) elif not s.value: matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T]) else: try: matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T]) except Exception, e: Log.error("", e)
def reverse(vals): # TODO: Test how to do this fastest l = len(vals) output = [None] * l for v in unwrap(vals): l -= 1 output[l] = v return wrap(output)
def argparse(defs): parser = _argparse.ArgumentParser() for d in listwrap(defs): args = d.copy() name = args.name args.name = None parser.add_argument(*unwrap(listwrap(name)), **args) namespace = parser.parse_args() output = {k: getattr(namespace, k) for k in vars(namespace)} return wrap(output)
def __init__(self, name, data, schema=None): #TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION data = list(unwrap(data)) Container.__init__(self, data, schema) if schema == None: self._schema = get_schema_from_list(data) else: self._schema = schema self.name = name self.data = data self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def params_pack(params, *args): settings = {} for a in args: for k, v in a.items(): k = unicode(k) if k in settings: continue settings[k] = v output = {str(k): unwrap(settings[k]) for k in params if k in settings} return output
def __init__(self, *args, **kwargs): """ CALLING Dict(**something) WILL RESULT IN A COPY OF something, WHICH IS UNLIKELY TO BE USEFUL. USE wrap() INSTEAD """ if DEBUG: d = _get(self, "_dict") for k, v in kwargs.items(): d[literal_field(k)] = unwrap(v) else: if args: args0 = args[0] if isinstance(args0, Mapping): _set(self, "_dict", args0) else: _set(self, "_dict", _get(args[0], "__dict__")) elif kwargs: _set(self, "_dict", unwrap(kwargs)) else: _set(self, "_dict", {})
def get(self, key): """ simple `select` """ if not _dictwrap: _late_import() return DictList(vals=[ unwrap(coalesce(_dictwrap(v), Null)[key]) for v in _get(self, "list") ])
def select(data, field_name): """ return list with values from field_name """ if isinstance(data, Cube): return data._select(_normalize_selects(field_name)) if isinstance(data, FlatList): return data.select(field_name) if isinstance(data, UniqueIndex): data = data._data.values( ) # THE SELECT ROUTINE REQUIRES dicts, NOT Dict WHILE ITERATING if isinstance(data, Mapping): return select_one(data, field_name) if isinstance(field_name, Mapping): field_name = wrap(field_name) if field_name.value in ["*", "."]: return data if field_name.value: # SIMPLIFY {"value":value} AS STRING field_name = field_name.value # SIMPLE PYTHON ITERABLE ASSUMED if isinstance(field_name, basestring): path = split_field(field_name) if len(path) == 1: return DictList([d[field_name] for d in data]) else: output = DictList() flat_list._select1(data, path, 0, output) return output elif isinstance(field_name, list): keys = [_select_a_field(wrap(f)) for f in field_name] return _select(Dict(), unwrap(data), keys, 0) else: keys = [_select_a_field(field_name)] return _select(Dict(), unwrap(data), keys, 0)
def es_aggop(es, mvel, query): select = listwrap(query.select) FromES = build_es_query(query) isSimple = AND(aggregates[s.aggregate] == "count" for s in select) if isSimple: return es_countop(es, query) # SIMPLE, USE TERMS FACET INSTEAD value2facet = dict() # ONLY ONE FACET NEEDED PER name2facet = dict() # MAP name TO FACET WITH STATS for s in select: if s.value not in value2facet: if is_keyword(s.value): unwrap(FromES.facets)[s.name] = { "statistical": { "field": s.value }, "facet_filter": simplify_esfilter(query.where) } else: unwrap(FromES.facets)[s.name] = { "statistical": { "script": es09.expressions.compile_expression(s.value, query) }, "facet_filter": simplify_esfilter(query.where) } value2facet[s.value] = s.name name2facet[s.name] = value2facet[s.value] data = es09.util.post(es, FromES, query.limit) matricies = { s.name: Matrix(value=fix_es_stats(data.facets[literal_field(s.name)])[ aggregates[s.aggregate]]) for s in select } cube = Cube(query.select, [], matricies) cube.frum = query return cube
def __setitem__(self, i, y): try: _list = _get(self, "list") if i <= len(_list): for i in range(len(_list), i): _list.append(None) _list[i] = unwrap(y) except Exception, e: if not _dictwrap: _late_import() _Log.error("problem", cause = e)
def __setitem__(self, i, y): try: _list = _get(self, "list") if i <= len(_list): for i in range(len(_list), i): _list.append(None) _list[i] = unwrap(y) except Exception, e: if not _dictwrap: _late_import() _Log.error("problem", cause=e)
def main(): try: settings = startup.read_settings() constants.set(settings.constants) Log.start(settings.debug) aws_args = dict( region_name=settings.aws.region, aws_access_key_id=unwrap(settings.aws.aws_access_key_id), aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key) ) ec2_conn = boto_ec2.connect_to_region(**aws_args) instances = _get_managed_instances(ec2_conn, settings.name) for i in instances: Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", insance_id=i.id, name=i.tags["Name"], ip=i.ip_address) _config_fabric(settings.fabric, i) _refresh_indexer() except Exception, e: Log.error("Problem with etl", e)
def select(data, field_name): """ return list with values from field_name """ if isinstance(data, Cube): return data._select(_normalize_selects(field_name)) if isinstance(data, FlatList): return data.select(field_name) if isinstance(data, UniqueIndex): data = data._data.values() # THE SELECT ROUTINE REQUIRES dicts, NOT Dict WHILE ITERATING if isinstance(data, Mapping): return select_one(data, field_name) if isinstance(field_name, Mapping): field_name = wrap(field_name) if field_name.value in ["*", "."]: return data if field_name.value: # SIMPLIFY {"value":value} AS STRING field_name = field_name.value # SIMPLE PYTHON ITERABLE ASSUMED if isinstance(field_name, basestring): path = split_field(field_name) if len(path) == 1: return DictList([d[field_name] for d in data]) else: output = DictList() flat_list._select1(data, path, 0, output) return output elif isinstance(field_name, list): keys = [_select_a_field(wrap(f)) for f in field_name] return _select(Dict(), unwrap(data), keys, 0) else: keys = [_select_a_field(field_name)] return _select(Dict(), unwrap(data), keys, 0)
def filter(data, where): """ where - a function that accepts (record, rownum, rows) and returns boolean """ if len(data) == 0 or where == None or where == TRUE_FILTER: return data if isinstance(data, Container): return data.filter(where) if isinstance(data, (list, set)): temp = jx_expression_to_function(where) dd = wrap(data) return wrap([unwrap(d) for i, d in enumerate(data) if temp(wrap(d), i, dd)]) else: Log.error("Do not know how to handle type {{type}}", type=data.__class__.__name__) try: return drill_filter(where, data) except Exception, _: # WOW! THIS IS INEFFICIENT! return wrap([unwrap(d) for d in drill_filter(where, [DictObject(d) for d in data])])
def __init__( self, aws_access_key_id=None, # CREDENTIAL aws_secret_access_key=None, # CREDENTIAL region=None, # NAME OF AWS REGION, REQUIRED FOR SOME BUCKETS settings=None): self.settings = settings try: if not settings.region: self.connection = boto.connect_s3( aws_access_key_id=unwrap(self.settings.aws_access_key_id), aws_secret_access_key=unwrap( self.settings.aws_secret_access_key)) else: self.connection = boto.s3.connect_to_region( self.settings.region, aws_access_key_id=unwrap(self.settings.aws_access_key_id), aws_secret_access_key=unwrap( self.settings.aws_secret_access_key)) except Exception, e: Log.error("Problem connecting to S3", e)
def _send_email(self): try: if self.accumulation: conn = connect_to_region( self.settings.region, aws_access_key_id=unwrap(self.settings.aws_access_key_id), aws_secret_access_key=unwrap( self.settings.aws_secret_access_key)) conn.send_email(source=self.settings.from_address, to_addresses=listwrap( self.settings.to_address), subject=self.settings.subject, body="\n\n".join(self.accumulation), format="text") conn.close() self.next_send = Date.now() + WAIT_TO_SEND_MORE self.accumulation = [] except Exception, e: self.next_send = Date.now() + WAIT_TO_SEND_MORE Log.warning("Could not send", e)
def __setattr__(self, key, value): if isinstance(key, str): ukey = key.decode("utf8") else: ukey = key d = self value = unwrap(value) if value is None: dict.pop(d, key, None) else: dict.__setitem__(d, ukey, value) return self
def __setattr__(self, key, value): if isinstance(key, str): ukey = key.decode("utf8") else: ukey = key d = _get(self, "_dict") value = unwrap(value) if value is None: d = _get(self, "_dict") d.pop(key, None) else: d[ukey] = value return self