def write_csv(out, json_list, header=False, shallow=False, keyset=None):
"""
Print a json list as csv. Rather than have to create an entire list in
memory, this prints out each element of the list individually.
:param out: The file in which to print (or stdout)
:param json_list: An iterable containing things that can be dumped by json
(typically lists or dicts)
:param header: Whether or not to include a csv header in the output
:param shallow: Whether or not to shallow parse. If True, only parse the
top level keys, and use the child value as the value,
event if it is a dictionary or list. Default is False.
:param keyset: A pre-populated KeySet object specifying how to write
the items in json_list to the csv file. Ignore this
if you want the KeySet to be generated for you.
"""
if not keyset:
keyset = csv.KeySet()
for item in json_list:
if not keyset:
for key, value in item.items():
json_key = csv.create_key(key, value, shallow)
keyset.add_key(json_key)
if header:
write_header_from_keyset(keyset, out)
values = []
for top_level_key in keyset.all_keys():
values = values + csv.dict_into_values(item, top_level_key, 0, shallow)
csv_string = csv.create_csv_string(values)
out.write(csv_string.encode('UTF-8') + '\n')
def write_csv(out, json_list, header=False, shallow=False, keyset=None):
"""
Print a json list as csv. Rather than have to create an entire list in
memory, this prints out each element of the list individually.
:param out: The file in which to print (or stdout)
:param json_list: An iterable containing things that can be dumped by json
(typically lists or dicts)
:param header: Whether or not to include a csv header in the output
:param shallow: Whether or not to shallow parse. If True, only parse the
top level keys, and use the child value as the value,
event if it is a dictionary or list. Default is False.
:param keyset: A pre-populated KeySet object specifying how to write
the items in json_list to the csv file. Ignore this
if you want the KeySet to be generated for you.
"""
if not keyset:
keyset = csv.KeySet()
for item in json_list:
if not keyset:
for key, value in item.items():
json_key = csv.create_key(key, value, shallow)
keyset.add_key(json_key)
if header:
write_header_from_keyset(keyset, out)
values = []
for top_level_key in keyset.all_keys():
values = values + csv.dict_into_values(item, top_level_key, 0,
shallow)
csv_string = csv.create_csv_string(values)
out.write(csv_string.encode('UTF-8') + '\n')
def _run():
"""
Run the function.
"""
with script_common.smart_open(tmp_path) as tmp:
key_set = csv.KeySet()
hash_map = {}
wrote_header = False
modify_time = False
for json_dict in json_list:
if not key_set:
for key, value in json_dict.items():
json_key = csv.create_key(key, value, shallow=True)
key_set.add_key(json_key)
try:
with open(old_path, 'r') as old:
first = True
reader = pycsv.DictReader(old)
for row in reader:
if first:
for key, value in row.items():
json_key = csv.create_key(key, value, shallow=True)
key_set.add_key(json_key)
script_output.write_header_from_keyset(key_set, tmp)
wrote_header = True
first = False
write_lookup_row(tmp, row, key_set, hash_map)
except IOError:
modify_time = time_key or modify_time
if not wrote_header:
script_output.write_header_from_keyset(key_set, tmp)
wrote_header = True
if modify_time:
json_dict[time_key] = "2000-01-01T00:01:01.000-00:00"
write_lookup_row(tmp, json_dict, key_set, hash_map)
if os.path.exists(old_path) and key_set:
os.remove(old_path)
if key_set:
shutil.move(tmp_path, old_path)
else:
os.remove(tmp_path)
def test_output_csv_given_keyset():
"""
Test that a csv write works when a KeySet is provided ahead of time.
"""
key_set = csv.KeySet()
sub_dict = {'meow': 'cat'}
data_dict = {'test': sub_dict}
for key, value in data_dict.items():
json_key = csv.create_key(key, value, shallow=True)
key_set.add_key(json_key)
data = [data_dict]
output_file = WriteTester()
script_output.write_csv(output_file, data, header=False, shallow=True, keyset=key_set)
lines = output_file.lines()
assert len(lines) == 1
assert lines[0] == '"{""meow"": ""cat""}"\n'
def test_output_csv_given_keyset():
"""
Test that a csv write works when a KeySet is provided ahead of time.
"""
key_set = csv.KeySet()
sub_dict = {'meow': 'cat'}
data_dict = {'test': sub_dict}
for key, value in data_dict.items():
json_key = csv.create_key(key, value, shallow=True)
key_set.add_key(json_key)
data = [data_dict]
output_file = WriteTester()
script_output.write_csv(output_file,
data,
header=False,
shallow=True,
keyset=key_set)
lines = output_file.lines()
assert len(lines) == 1
assert lines[0] == '"{""meow"": ""cat""}"\n'
