def test_one_level_deep_flattens(self):
data = dict(flat1=1, dict1=dict(c=1, d=2))
result = nested_to_record(data)
expected = {'dict1.c': 1, 'dict1.d': 2, 'flat1': 1}
self.assertEqual(result, expected)
def test_flat_stays_flat(self):
recs = [dict(flat1=1, flat2=2),
dict(flat1=3, flat2=4),
]
result = nested_to_record(recs)
expected = recs
self.assertEqual(result, expected)
def test_flat_stays_flat(self):
recs = [dict(flat1=1,flat2=2),
dict(flat1=3,flat2=4),
]
result = nested_to_record(recs)
expected = recs
self.assertEqual(result, expected)
def json_to_norm_dict(s, c):
logging.info(str(c))
normalized_dict = pjson.nested_to_record(json.loads(s))
return {
key.replace('.', '_'): value
for key, value in normalized_dict.iteritems()
if key in json_data_columns.WANTED_COLUMNS_STOCKTWITS
}
def test_one_level_deep_flattens(self):
data = dict(flat1=1,
dict1=dict(c=1,d=2))
result = nested_to_record(data)
expected = {'dict1.c': 1,
'dict1.d': 2,
'flat1': 1}
self.assertEqual(result,expected)
def test_nested_flattens(self):
data = dict(flat1=1,
dict1=dict(c=1, d=2),
nested=dict(e=dict(c=1, d=2),
d=2))
result = nested_to_record(data)
expected = {'dict1.c': 1,
'dict1.d': 2,
'flat1': 1,
'nested.d': 2,
'nested.e.c': 1,
'nested.e.d': 2}
self.assertEqual(result, expected)
def test_nested_flattens(self):
data = dict(flat1=1,
dict1=dict(c=1,d=2),
nested=dict(e=dict(c=1,d=2),
d=2))
result = nested_to_record(data)
expected = {'dict1.c': 1,
'dict1.d': 2,
'flat1': 1,
'nested.d': 2,
'nested.e.c': 1,
'nested.e.d': 2}
self.assertEqual(result,expected)
def read_nested_write_flat(infile, outfile):
with open(infile, 'r') as infile:
with open(outfile, 'a') as outfile:
for record in read_jsonl_stream(infile):
dump_jsonl_stream(nested_to_record(record), outfile)
print "saved records to {}".format(outfile.name)
