def _run_test(self, config):
"""
:param config: list of {name: nature} objects
:return: test function
"""
generator = make_const
for c in reversed(config):
for name, rep_type in items(c)[:1]:
generator = rep_type_to_generator[rep_type](name, generator)
schema = SchemaTree(locked=True)
path = []
for c in config:
for name, rep_type in items(c)[:1]:
path.append(name)
schema.add(join_field(path), rep_type, int)
# THESE TESTS ASSUME ONLY ONE LEAF
full_name = join_field([name for c in config for name, rep_type in items(c)[:1]])
data, values, rep_level, def_level = zip(*list(generator()))
expected_values = {full_name: sum(values, [])}
expected_reps = {full_name: sum(rep_level, [])}
expected_defs = {full_name: sum(def_level, [])}
table = rows_to_columns(list(data), schema)
self.assertEqual(table.values, expected_values)
self.assertEqual(table.reps, expected_reps)
self.assertEqual(table.defs, expected_defs)
def _scrub(result):
if isinstance(result, text_type):
return result
elif isinstance(result, binary_type):
return result.decode('utf8')
elif isinstance(result, number_types):
return result
elif not result:
return {}
elif isinstance(result, (list, ParseResults)):
if not result:
return None
elif len(result) == 1:
return _scrub(result[0])
else:
output = [rr for r in result for rr in [_scrub(r)] if rr != None]
# IF ALL MEMBERS OF A LIST ARE LITERALS, THEN MAKE THE LIST LITERAL
if all(
isinstance(r, Mapping) and "literal" in r.keys()
for r in output):
output = {"literal": [r['literal'] for r in output]}
return output
elif not items(result):
return {}
else:
return {
k: vv
for k, v in result.items() for vv in [_scrub(v)] if vv != None
}
def _scrub(result):
if isinstance(result, text_type):
return result
elif isinstance(result, binary_type):
return result.decode('utf8')
elif isinstance(result, number_types):
return result
elif not result:
return {}
elif isinstance(result, (list, ParseResults)):
if not result:
return None
elif len(result) == 1:
return _scrub(result[0])
else:
output = [
rr
for r in result
for rr in [_scrub(r)]
if rr != None
]
# IF ALL MEMBERS OF A LIST ARE LITERALS, THEN MAKE THE LIST LITERAL
if all(isinstance(r, Mapping) and "literal" in r.keys() for r in output):
output = {"literal": [r['literal'] for r in output]}
return output
elif not items(result):
return {}
else:
return {
k: vv
for k, v in result.items()
for vv in [_scrub(v)]
if vv != None
}
def expand_json(rows):
# CONVERT JSON TO VALUES
for r in rows:
for k, json in items(r):
if isinstance(json, text) and json[0:1] in ("[", "{"):
with suppress_exception:
value = mo_json.json2value(json)
r[k] = value
def expand_json(rows):
# CONVERT JSON TO VALUES
for r in rows:
for k, json in items(r):
if isinstance(json, text_type) and json[0:1] in ("[", "{"):
with suppress_exception:
value = mo_json.json2value(json)
r[k] = value
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today() - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception as e:
Log.warning("could not delete index {{index}}", index=c.index, cause=e)
for t, q in items(self.known_queues):
if unix2Date(t) + self.rollover_interval < Date.today() - self.rollover_max:
with self.locker:
del self.known_queues[t]
pass
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today() - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception as e:
Log.warning("could not delete index {{index}}", index=c.index, cause=e)
for t, q in items(self.known_queues):
if unix2Date(t) + self.rollover_interval < Date.today() - self.rollover_max:
with self.locker:
del self.known_queues[t]
pass
bool: 2,
int: 3,
long: 3,
Date: 4,
float: 5,
text_type: 6,
binary_type: 6,
object: 7,
dict: 8,
Mapping: 9,
Data: 10,
list: 11,
FlatList: 12,
}
for k, v in items(_merge_order):
_merge_order[k.__name__] = v
def _merge_python_type(A, B):
a = _merge_order[A]
b = _merge_order[B]
if a >= b:
output = A
else:
output = B
if isinstance(output, str):
return output
else:
none_type: OBJECT,
Data: OBJECT,
dict: OBJECT,
object: OBJECT,
list: NESTED,
set: NESTED,
# tuple: NESTED, # DO NOT INCLUDE, WILL HIDE LOGIC ERRORS
FlatList: NESTED,
Date: NUMBER
}
if PY2:
python_type_to_json_type[str] = STRING
python_type_to_json_type[long] = NUMBER
for k, v in items(python_type_to_json_type):
python_type_to_json_type[k.__name__] = v
_merge_order = {
BOOLEAN: 1,
INTEGER: 2,
NUMBER: 3,
STRING: 4,
OBJECT: 5,
NESTED: 6
}
def _merge_json_type(A, B):
a = _merge_order[A]
b = _merge_order[B]
NullType: 1,
bool: 2,
int: 3,
Date: 4,
float: 5,
text_type: 6,
binary_type: 6,
object: 7,
dict: 8,
Mapping: 9,
Data: 10,
list: 11,
FlatList: 12,
}
for k, v in items(_merge_order):
_merge_order[k.__name__] = v
def _merge_python_type(A, B):
a = _merge_order[A]
b = _merge_order[B]
if a >= b:
output = A
else:
output = B
if isinstance(output, str):
return output
else:
