def joindicts(dest, *dictlist): """Merges one or more dictionaries into dest recursively. Dictionaries are merged, lists are merged. Scalars and strings are ignored. Returns the generated result. To merge with overwrite, use the native dict update method. """ result = dest for inputdict in dictlist: for key, value in inputdict.items(): # If this is a new value, take as is if key not in result: result[key] = value continue # Handle collision existing_value = dest[key] if typecheck.is_dict(existing_value) and typecheck.is_dict(value): result[key] = joindicts(existing_value, value) elif typecheck.is_list(existing_value) and typecheck.is_list( value): result[key] = existing_value + value else: raise Exception("Failed to merge dictionaries: " + f"could not resolve collision on key '{key}'") return result
def flatten_dict(d, prefix='', result=None): "Flattens a dictionary using path separators" # recursive algorithm where each pass gives the result object to the next patch # The key algorithm boils down to: # - If its not a scalar (dict or list), build the prefix # - If its a scalar (string or number), assign the value result = {} if result is None else result if typecheck.is_dict(d): # dictionaries add a / behind them for non-root ones if prefix: prefix = prefix + '/' for key, value in d.items(): if typecheck.is_scalar(value): result[f"{prefix}{key}"] = value else: flatten_dict(value, prefix=prefix + key, result=result) elif typecheck.is_list(d): for idx, value in enumerate(d): if typecheck.is_scalar(value): result[f'{prefix}[{idx}]'] = value else: flatten_dict(value, prefix=f'{prefix}[{idx}]', result=result) else: raise Exception('Unsupported type ' + str(type(d))) return result
def group_fields(obj, groups=[]): "Returns a new dictionary where the items that start with groupname_ are consolidated" if not typecheck.is_list(groups): raise TypeError("groups needs to be a list or tuple") groups = check_not_grouped(obj, groups) result = {} for key, value in obj.items(): group_results = list(filter(lambda g: key.startswith(g + '_'), groups)) if not group_results: result[key] = value continue group_name = group_results[0] subkey = key[len(group_name) + 1:] group = result.setdefault(group_name, {}) group[subkey] = value return result
def write_dicts_artifact(filename, lines: typing.Iterable[dict], autoflatten=True): "Basically just writes a raw list of dictionaries as a csv" if autoflatten: oldlines = lines lines = [] for line in oldlines: new_line = {} for key, value in line.items(): if typecheck.is_list(value): for i in range(len(value)): new_line[f"{key}_{i+1}"] = value[i] else: new_line[key] = value lines.append(new_line) basepath = path.join(path.dirname(__file__), '../../../artifacts/') os.makedirs(basepath, exist_ok=True) save_csv(lines, path.join(basepath, filename))