def joindicts(dest, *dictlist):
"""Merges one or more dictionaries into dest recursively.
Dictionaries are merged, lists are merged. Scalars and strings are ignored.
Returns the generated result.
To merge with overwrite, use the native dict update method.
"""
result = dest
for inputdict in dictlist:
for key, value in inputdict.items():
# If this is a new value, take as is
if key not in result:
result[key] = value
continue
# Handle collision
existing_value = dest[key]
if typecheck.is_dict(existing_value) and typecheck.is_dict(value):
result[key] = joindicts(existing_value, value)
elif typecheck.is_list(existing_value) and typecheck.is_list(
value):
result[key] = existing_value + value
else:
raise Exception("Failed to merge dictionaries: " +
f"could not resolve collision on key '{key}'")
return result
def flatten_dict(d, prefix='', result=None):
"Flattens a dictionary using path separators"
# recursive algorithm where each pass gives the result object to the next patch
# The key algorithm boils down to:
# - If its not a scalar (dict or list), build the prefix
# - If its a scalar (string or number), assign the value
result = {} if result is None else result
if typecheck.is_dict(d):
# dictionaries add a / behind them for non-root ones
if prefix:
prefix = prefix + '/'
for key, value in d.items():
if typecheck.is_scalar(value):
result[f"{prefix}{key}"] = value
else:
flatten_dict(value, prefix=prefix + key, result=result)
elif typecheck.is_list(d):
for idx, value in enumerate(d):
if typecheck.is_scalar(value):
result[f'{prefix}[{idx}]'] = value
else:
flatten_dict(value, prefix=f'{prefix}[{idx}]', result=result)
else:
raise Exception('Unsupported type ' + str(type(d)))
return result
def group_fields(obj, groups=[]):
"Returns a new dictionary where the items that start with groupname_ are consolidated"
if not typecheck.is_list(groups):
raise TypeError("groups needs to be a list or tuple")
groups = check_not_grouped(obj, groups)
result = {}
for key, value in obj.items():
group_results = list(filter(lambda g: key.startswith(g + '_'), groups))
if not group_results:
result[key] = value
continue
group_name = group_results[0]
subkey = key[len(group_name) + 1:]
group = result.setdefault(group_name, {})
group[subkey] = value
return result
def write_dicts_artifact(filename,
lines: typing.Iterable[dict],
autoflatten=True):
"Basically just writes a raw list of dictionaries as a csv"
if autoflatten:
oldlines = lines
lines = []
for line in oldlines:
new_line = {}
for key, value in line.items():
if typecheck.is_list(value):
for i in range(len(value)):
new_line[f"{key}_{i+1}"] = value[i]
else:
new_line[key] = value
lines.append(new_line)
basepath = path.join(path.dirname(__file__), '../../../artifacts/')
os.makedirs(basepath, exist_ok=True)
save_csv(lines, path.join(basepath, filename))
