def save_data_csv(self,
location,
data_map,
*,
lang='en',
nest_additional=[],
groups=[],
key=None,
fields=None):
"""Write a DataMap to a location in the data directory.
If key is given, then the saving is restricted to what's inside that key.
If fields are given, only fields within the list are exported.
At least one of key or fields is required.
TODO: Write about nest_additional and groups
"""
location = self.get_data_path(location)
extracted = extract_sub_data(data_map,
key=key,
fields=fields,
lang=lang)
flattened_rows = flatten(extracted,
nest=['name_' + lang] + nest_additional)
flattened_rows = [
ungroup_fields(v, groups=groups) for v in flattened_rows
]
save_csv(flattened_rows, location)
def save_base_map_csv(self, location, base_map, *, groups=['name']):
location = self.get_data_path(location)
if 'name' not in groups:
raise Exception("Name is a required group for base maps")
rows = base_map.to_list()
rows = [ungroup_fields(v, groups=groups) for v in rows]
save_csv(rows, location)
def dump_gmd(filename: str):
base_name = filename
if base_name.lower().endswith('.gmd'):
base_name = base_name[:-8]
gmd_data = load_text(base_name)
rows = []
for idx, (key, entries) in enumerate(gmd_data.keyed_entries.items()):
rows.append({
'index': idx,
'key': key,
**entries
})
output = StringIO()
save_csv(rows, output)
return output.getvalue()
def write_dicts_artifact(filename,
lines: typing.Iterable[dict],
autoflatten=True):
"Basically just writes a raw list of dictionaries as a csv"
if autoflatten:
oldlines = lines
lines = []
for line in oldlines:
new_line = {}
for key, value in line.items():
if typecheck.is_list(value):
for i in range(len(value)):
new_line[f"{key}_{i+1}"] = value[i]
else:
new_line[key] = value
lines.append(new_line)
basepath = path.join(path.dirname(__file__), '../../../artifacts/')
os.makedirs(basepath, exist_ok=True)
save_csv(lines, path.join(basepath, filename))
def save_csv(self, location, rows, *, schema=None):
"Saves a raw csv relative to the source data location"
if schema:
rows, errors = schema.dump(rows, many=True)
location = self.get_data_path(location)
save_csv(rows, location)
def save_csv(self, location, rows):
"Saves a raw csv relative to the source data location"
location = self.get_data_path(location)
save_csv(rows, location)