def save_data_csv(self,
location,
data_map,
*,
lang='en',
nest_additional=[],
groups=[],
key=None,
fields=None):
"""Write a DataMap to a location in the data directory.
If key is given, then the saving is restricted to what's inside that key.
If fields are given, only fields within the list are exported.
At least one of key or fields is required.
TODO: Write about nest_additional and groups
"""
extracted = data_map.extract(key=key, fields=fields, lang=lang)
flattened_rows = flatten(extracted,
nest=['name_' + lang] + nest_additional)
flattened_rows = [
ungroup_fields(v, groups=groups) for v in flattened_rows
]
self.save_csv(location, flattened_rows)
def save_base_map_csv(self, location, base_map, *, groups=['name']):
if 'name' not in groups:
raise Exception("Name is a required group for base maps")
rows = base_map.to_list()
rows = [ungroup_fields(v, groups=groups) for v in rows]
self.save_csv(location, rows)
def save_data_csv(self,
location,
data_map: DataMap,
*,
key_join='name_en',
nest_additional=[],
groups=[],
key=None,
fields=None,
schema=None):
"""Write a DataMap to a location in the data directory.
If key is given, then the saving is restricted to what's inside that key.
If fields are given, only fields within the list are exported.
At least one of key or fields is required.
TODO: Write about nest_additional and groups
"""
if not key and not fields:
raise ValueError("Either a key or a list of fields " +
"must be given when persisting a data map")
flattened_rows = []
for entry in data_map.values():
ex_values = {k: entry[k] for k in data_map.keys_ex}
row_base = {'base_' + key_join: entry[key_join], **ex_values}
items = None
if key and key in entry and entry[key]:
items = entry[key]
if not typecheck.is_flat_iterable(items):
items = [items]
elif not key and fields:
items = [entry.to_dict()]
if not items:
continue
for item in items:
# If fields is given, extract them
if fields:
values = extract_fields(item, *fields)
else:
values = copy.deepcopy(item)
new_row = {**row_base, **values}
flattened_rows.append(new_row)
flattened_rows = [
ungroup_fields(v, groups=groups) for v in flattened_rows
]
self.save_csv(location, flattened_rows, schema=schema)
def save_base_map_csv(self,
location,
base_map: DataMap,
*,
groups=['name'],
schema=None,
translation_filename=None,
translation_extra=[],
key_join='name_en'):
"""
Saves a base map as a csv file.
If a marshmallow schema is provided, groups is ignored.
The schema becomes in charge of flattening.
"""
if 'name' not in groups:
raise Exception("Name is a required group for base maps")
# Write translation file
if translation_filename:
translations = []
translation_fields = ['name'] + translation_extra
for row in base_map.values():
ex_values = {k: row[k] for k in base_map.keys_ex}
translation_row = {key_join: row[key_join], **ex_values}
for lang in self.languages:
for field in translation_fields:
field_key = f'{field}_{lang}'
if field_key in translation_row:
continue
value = row.get(field, {}).get(lang, '') or ''
translation_row[field_key] = value.strip()
translations.append(translation_row)
# Remove separated entries from base map
# Removes all "extra group fields" and non-english name entries
row['name'] = {'en': row['name']['en']}
for field in translation_extra:
if field in row: del row[field]
self.save_csv(translation_filename, translations)
rows = base_map.to_list()
if not schema:
rows = [ungroup_fields(v, groups=groups) for v in rows]
else:
rows, errors = schema.dump(rows, many=True)
self.save_csv(location, rows)
def ungroup_fields(self, data):
groups = list(self.__groups__ or []) + self.identify_prefixes()
result = ungroup_fields(data, groups=groups)
# Now weave the translation fields at the end
translation_groups = list(self.__translation_groups__ or [])
for lang in cfg.all_languages:
for field in translation_groups:
field_value = result[field]
try:
result[f"{field}_{lang}"] = field_value[lang]
except KeyError:
raise KeyError(
f'No language entry {lang} for field {field}')
for field in translation_groups:
del result[field]
return result
def save_base_map_csv(self,
location,
base_map,
*,
groups=['name'],
schema=None):
"""
Saves a base map as a csv file.
If a marshmallow schema is provided, groups is ignored.
The schema becomes in charge of flattening.
"""
if 'name' not in groups:
raise Exception("Name is a required group for base maps")
rows = base_map.to_list()
if not schema:
rows = [ungroup_fields(v, groups=groups) for v in rows]
self.save_csv(location, rows, schema=schema)
def save_base_map_csv(self, location, base_map, *, groups=['name'], schema=None, translation_filename=None, translation_extra=[]):
"""
Saves a base map as a csv file.
If a marshmallow schema is provided, groups is ignored.
The schema becomes in charge of flattening.
"""
if 'name' not in groups:
raise Exception("Name is a required group for base maps")
rows = base_map.to_list()
if translation_filename:
translations = []
translation_fields = ['name'] + translation_extra
for row in rows:
translation_row = {}
for lang in self.languages:
for field in translation_fields:
value = row[field].get(lang, '') or ''
translation_row[f'{field}_{lang}'] = value.strip()
translations.append(translation_row)
row['name'] = { 'en': row['name']['en'] }
for field in translation_extra:
del row[field]
self.save_csv(translation_filename, translations)
if not schema:
rows = [ungroup_fields(v, groups=groups) for v in rows]
else:
rows, errors = schema.dump(rows, many=True)
self.save_csv(location, rows)
def ungroup_fields(self, data):
groups = list(self.__groups__ or []) + self.identify_prefixes()
result = ungroup_fields(data, groups=groups)
return result
def ungroup_fields(self, data):
groups = self.__groups__ or []
return ungroup_fields(data, groups=groups)
