def write_csv(out, json_list, header=False, shallow=False, keyset=None): """ Print a json list as csv. Rather than have to create an entire list in memory, this prints out each element of the list individually. :param out: The file in which to print (or stdout) :param json_list: An iterable containing things that can be dumped by json (typically lists or dicts) :param header: Whether or not to include a csv header in the output :param shallow: Whether or not to shallow parse. If True, only parse the top level keys, and use the child value as the value, event if it is a dictionary or list. Default is False. :param keyset: A pre-populated KeySet object specifying how to write the items in json_list to the csv file. Ignore this if you want the KeySet to be generated for you. """ if not keyset: keyset = csv.KeySet() for item in json_list: if not keyset: for key, value in item.items(): json_key = csv.create_key(key, value, shallow) keyset.add_key(json_key) if header: write_header_from_keyset(keyset, out) values = [] for top_level_key in keyset.all_keys(): values = values + csv.dict_into_values(item, top_level_key, 0, shallow) csv_string = csv.create_csv_string(values) out.write(csv_string.encode('UTF-8') + '\n')
def _run(): """ Run the function. """ with script_common.smart_open(tmp_path) as tmp: key_set = csv.KeySet() hash_map = {} wrote_header = False modify_time = False for json_dict in json_list: if not key_set: for key, value in json_dict.items(): json_key = csv.create_key(key, value, shallow=True) key_set.add_key(json_key) try: with open(old_path, 'r') as old: first = True reader = pycsv.DictReader(old) for row in reader: if first: for key, value in row.items(): json_key = csv.create_key(key, value, shallow=True) key_set.add_key(json_key) script_output.write_header_from_keyset(key_set, tmp) wrote_header = True first = False write_lookup_row(tmp, row, key_set, hash_map) except IOError: modify_time = time_key or modify_time if not wrote_header: script_output.write_header_from_keyset(key_set, tmp) wrote_header = True if modify_time: json_dict[time_key] = "2000-01-01T00:01:01.000-00:00" write_lookup_row(tmp, json_dict, key_set, hash_map) if os.path.exists(old_path) and key_set: os.remove(old_path) if key_set: shutil.move(tmp_path, old_path) else: os.remove(tmp_path)
def test_output_csv_given_keyset(): """ Test that a csv write works when a KeySet is provided ahead of time. """ key_set = csv.KeySet() sub_dict = {'meow': 'cat'} data_dict = {'test': sub_dict} for key, value in data_dict.items(): json_key = csv.create_key(key, value, shallow=True) key_set.add_key(json_key) data = [data_dict] output_file = WriteTester() script_output.write_csv(output_file, data, header=False, shallow=True, keyset=key_set) lines = output_file.lines() assert len(lines) == 1 assert lines[0] == '"{""meow"": ""cat""}"\n'