def _run_test(self, config): """ :param config: list of {name: nature} objects :return: test function """ generator = make_const for c in reversed(config): for name, rep_type in items(c)[:1]: generator = rep_type_to_generator[rep_type](name, generator) schema = SchemaTree(locked=True) path = [] for c in config: for name, rep_type in items(c)[:1]: path.append(name) schema.add(join_field(path), rep_type, int) # THESE TESTS ASSUME ONLY ONE LEAF full_name = join_field([name for c in config for name, rep_type in items(c)[:1]]) data, values, rep_level, def_level = zip(*list(generator())) expected_values = {full_name: sum(values, [])} expected_reps = {full_name: sum(rep_level, [])} expected_defs = {full_name: sum(def_level, [])} table = rows_to_columns(list(data), schema) self.assertEqual(table.values, expected_values) self.assertEqual(table.reps, expected_reps) self.assertEqual(table.defs, expected_defs)
def _scrub(result): if isinstance(result, text_type): return result elif isinstance(result, binary_type): return result.decode('utf8') elif isinstance(result, number_types): return result elif not result: return {} elif isinstance(result, (list, ParseResults)): if not result: return None elif len(result) == 1: return _scrub(result[0]) else: output = [rr for r in result for rr in [_scrub(r)] if rr != None] # IF ALL MEMBERS OF A LIST ARE LITERALS, THEN MAKE THE LIST LITERAL if all( isinstance(r, Mapping) and "literal" in r.keys() for r in output): output = {"literal": [r['literal'] for r in output]} return output elif not items(result): return {} else: return { k: vv for k, v in result.items() for vv in [_scrub(v)] if vv != None }
def _scrub(result): if isinstance(result, text_type): return result elif isinstance(result, binary_type): return result.decode('utf8') elif isinstance(result, number_types): return result elif not result: return {} elif isinstance(result, (list, ParseResults)): if not result: return None elif len(result) == 1: return _scrub(result[0]) else: output = [ rr for r in result for rr in [_scrub(r)] if rr != None ] # IF ALL MEMBERS OF A LIST ARE LITERALS, THEN MAKE THE LIST LITERAL if all(isinstance(r, Mapping) and "literal" in r.keys() for r in output): output = {"literal": [r['literal'] for r in output]} return output elif not items(result): return {} else: return { k: vv for k, v in result.items() for vv in [_scrub(v)] if vv != None }
def expand_json(rows): # CONVERT JSON TO VALUES for r in rows: for k, json in items(r): if isinstance(json, text) and json[0:1] in ("[", "{"): with suppress_exception: value = mo_json.json2value(json) r[k] = value
def expand_json(rows): # CONVERT JSON TO VALUES for r in rows: for k, json in items(r): if isinstance(json, text_type) and json[0:1] in ("[", "{"): with suppress_exception: value = mo_json.json2value(json) r[k] = value
def _delete_old_indexes(self, candidates): for c in candidates: timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S") if timestamp + self.rollover_interval < Date.today() - self.rollover_max: # Log.warning("Will delete {{index}}", index=c.index) try: self.cluster.delete_index(c.index) except Exception as e: Log.warning("could not delete index {{index}}", index=c.index, cause=e) for t, q in items(self.known_queues): if unix2Date(t) + self.rollover_interval < Date.today() - self.rollover_max: with self.locker: del self.known_queues[t] pass
bool: 2, int: 3, long: 3, Date: 4, float: 5, text_type: 6, binary_type: 6, object: 7, dict: 8, Mapping: 9, Data: 10, list: 11, FlatList: 12, } for k, v in items(_merge_order): _merge_order[k.__name__] = v def _merge_python_type(A, B): a = _merge_order[A] b = _merge_order[B] if a >= b: output = A else: output = B if isinstance(output, str): return output else:
none_type: OBJECT, Data: OBJECT, dict: OBJECT, object: OBJECT, list: NESTED, set: NESTED, # tuple: NESTED, # DO NOT INCLUDE, WILL HIDE LOGIC ERRORS FlatList: NESTED, Date: NUMBER } if PY2: python_type_to_json_type[str] = STRING python_type_to_json_type[long] = NUMBER for k, v in items(python_type_to_json_type): python_type_to_json_type[k.__name__] = v _merge_order = { BOOLEAN: 1, INTEGER: 2, NUMBER: 3, STRING: 4, OBJECT: 5, NESTED: 6 } def _merge_json_type(A, B): a = _merge_order[A] b = _merge_order[B]
NullType: 1, bool: 2, int: 3, Date: 4, float: 5, text_type: 6, binary_type: 6, object: 7, dict: 8, Mapping: 9, Data: 10, list: 11, FlatList: 12, } for k, v in items(_merge_order): _merge_order[k.__name__] = v def _merge_python_type(A, B): a = _merge_order[A] b = _merge_order[B] if a >= b: output = A else: output = B if isinstance(output, str): return output else: