def init_lg_repro_drop_data(drop: dict): """ Creates and appends per-drop reproducibility information at the logical graph stage. :param drop: :return: The same drop with appended reproducibility information """ level = rflag_caster(drop["reprodata"]["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: data = accumulate_lg_drop_data(drop, rmode) merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root drop["reprodata"][rmode.name]["lg_data"] = data drop["reprodata"][rmode.name]["lg_parenthashes"] = {} else: data = accumulate_lg_drop_data(drop, level) merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root drop["reprodata"]["lg_data"] = data drop["reprodata"]["lg_parenthashes"] = {} return drop
def accumulate_pgt_unroll_drop_data(drop: dict): """ Accumulates relevant reproducibility fields for a single drop at the physical template level. :param drop: :return: A dictionary containing accumulated reproducibility data for a given drop. """ data = {} rmode = rflag_caster(drop['reprodata']['rmode']) if not rmode_supported(rmode): logger.warning('Requested reproducibility mode %s not yet implemented', str(rmode)) rmode = REPRO_DEFAULT drop['reprodata']['rmode'] = str(rmode.value) if rmode == ReproducibilityFlags.NOTHING: return data if rmode == ReproducibilityFlags.REPRODUCE: data['type'] = drop['type'] if drop['type'] == 'plain': data['storage'] = drop['storage'] return data if rmode.value >= ReproducibilityFlags.RERUN.value: data['type'] = drop['type'] if data['type'] == 'plain': data['storage'] = drop['storage'] else: data['dt'] = drop[ 'dt'] # WARNING: Added to differentiate between subtle component differences. if rmode == ReproducibilityFlags.RECOMPUTE or rmode == ReproducibilityFlags.REPLICATE_COMP: data['rank'] = drop['rank'] return data
def init_pg_repro_data(physical_graph: list): """ Handles adding reproducibility data at the physical graph template level. :param physical_graph: The logical graph data structure (a list of drops + reprodata dictionary) :return: The same pg object with new information appended """ reprodata = physical_graph.pop() if "rmode" not in reprodata: physical_graph.append(reprodata) return physical_graph level = rflag_caster(reprodata["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT if level == ReproducibilityFlags.NOTHING: physical_graph.append(reprodata) return physical_graph for drop in physical_graph: init_pg_repro_drop_data(drop) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: leaves, _ = build_blockdag(physical_graph, "pg", rmode) reprodata[rmode.name]["signature"] = agglomerate_leaves(leaves) else: leaves, _ = build_blockdag(physical_graph, "pg") reprodata["signature"] = agglomerate_leaves(leaves) physical_graph.append(reprodata) logger.info("Reproducibility data finished at PG level") return physical_graph
def init_lg_repro_data(logical_graph: dict): """ Handles adding reproducibility data at the logical graph level. Also builds the logical data blockdag over the entire structure. :param logical_graph: The logical graph data structure (a JSON object (a dict)) :return: The same lgt object with new information appended """ if "reprodata" not in logical_graph: return logical_graph level = rflag_caster(logical_graph["reprodata"]["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT if level == ReproducibilityFlags.NOTHING: return logical_graph for drop in logical_graph.get("nodeDataArray", []): init_lg_repro_drop_data(drop) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: if rmode.name not in logical_graph["reprodata"]: logical_graph["reprodata"][rmode.name] = {} leaves, _ = lg_build_blockdag(logical_graph, rmode) logical_graph["reprodata"][ rmode.name]["signature"] = agglomerate_leaves(leaves) else: leaves, _ = lg_build_blockdag(logical_graph) logical_graph["reprodata"]["signature"] = agglomerate_leaves(leaves) logger.info("Reproducibility data finished at LG level") return logical_graph
def accumulate_pgt_unroll_drop_data(drop: dict): """ Accumulates relevant reproducibility fields for a single drop at the physical template level. :param drop: :return: A dictionary containing accumulated reproducibility data for a given drop. """ if drop.get("reprodata") is None: drop["reprodata"] = { "rmode": str(REPRO_DEFAULT.value), "lg_blockhash": None } if drop["reprodata"].get("rmode") is None: level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) else: level = rflag_caster(drop["reprodata"]["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) if drop.get("type") is None: return {} drop_type = drop["type"] if level == ReproducibilityFlags.ALL: data = {} for rmode in ALL_RMODES: pgt_fields = pgt_unroll_block_fields(drop_type, rmode) data[rmode.name] = extract_fields(drop, pgt_fields) else: pgt_fields = pgt_unroll_block_fields(drop_type, level) data = extract_fields(drop, pgt_fields) return data
def init_lgt_repro_drop_data(drop: dict, level: ReproducibilityFlags): """ Creates and appends per-drop reproducibility information at the logical template stage. :param drop: :param level: :return: The same drop with appended reproducibility information. """ # Catch pre-set per-drop rmode if "reprodata" in drop.keys(): if "rmode" in drop["reprodata"].keys(): level = rflag_caster(drop["reprodata"]["rmode"]) else: drop["reprodata"] = {"rmode": str(level.value)} if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: data = accumulate_lgt_drop_data(drop, rmode) merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root drop["reprodata"][rmode.name] = { "rmode": str(rmode.value), "lgt_data": data, "lg_parenthashes": {}, } else: data = accumulate_lgt_drop_data(drop, level) merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root drop["reprodata"] = { "rmode": str(level.value), "lgt_data": data, "lg_parenthashes": {}, } return drop
def accumulate_pgt_partition_drop_data(drop: dict): """ Is as combination of unroll drop data :param drop: :return: """ if drop.get("reprodata") is None: drop["reprodata"] = { "rmode": str(REPRO_DEFAULT.value), "lg_blockhash": None } if drop["reprodata"].get("rmode") is None: level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) else: level = rflag_caster(drop["reprodata"]["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) if level == ReproducibilityFlags.ALL: data = {} unroll_data = accumulate_pgt_unroll_drop_data(drop) for rmode in ALL_RMODES: pgt_fields = pgt_partition_block_fields(rmode) data[rmode.name] = extract_fields(drop, pgt_fields) unroll_data[rmode.name].update(data[rmode.name]) return unroll_data else: pgt_fields = pgt_partition_block_fields(level) data = extract_fields(drop, pgt_fields) return_data = accumulate_pgt_unroll_drop_data(drop) return_data.update(data) return return_data
def init_runtime_repro_data(runtime_graph: dict, reprodata: dict): """ Adds reproducibility data at the runtime level to graph-wide values. :param runtime_graph: :param reprodata: :return: """ if reprodata is None: return runtime_graph level = rflag_caster(reprodata["rmode"]) if not rmode_supported(level): # TODO: Logging needs sessionID at this stage # logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) level = REPRO_DEFAULT reprodata["rmode"] = str(level.value) for drop in runtime_graph.values(): init_rg_repro_drop_data(drop) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: leaves, _ = build_blockdag(list(runtime_graph.values()), "rg", rmode) reprodata[rmode.name]["signature"] = agglomerate_leaves(leaves) else: leaves, _ = build_blockdag(list(runtime_graph.values()), "rg") reprodata["signature"] = agglomerate_leaves(leaves) runtime_graph["reprodata"] = reprodata # logger.info("Reproducibility data finished at runtime level") return runtime_graph
def init_lgt_repro_data(logical_graph_template: dict, rmode: str): """ Creates and appends graph-wide reproducibility data at the logical template stage. Currently, this is basically a stub that adds the requested flag to the graph. Later, this will contain significantly more information. :param logical_graph_template: The logical graph data structure (a JSON object (a dict)) :param rmode: One several values 0-5 defined in constants.py :return: The same lgt object with new information appended """ rmode = rflag_caster(rmode) if not rmode_supported(rmode): logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT if rmode == ReproducibilityFlags.NOTHING: return logical_graph_template reprodata = { "rmode": str(rmode.value), "meta_data": accumulate_meta_data() } meta_tree = MerkleTree(reprodata.items(), common_hash) reprodata["merkleroot"] = meta_tree.merkle_root for drop in logical_graph_template.get("nodeDataArray", []): init_lgt_repro_drop_data(drop, rmode) logical_graph_template["reprodata"] = reprodata logger.info("Reproducibility data finished at LGT level") return logical_graph_template
def init_rg_repro_drop_data(drop: dict): """ Creates and appends per-drop reproducibility information at the runtime graph stage. :param drop: :return: The same drop with appended reproducibility information """ level = rflag_caster(drop["reprodata"]["rmode"]) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: drop["reprodata"][rmode.name]["rg_parenthashes"] = {} elif level != ReproducibilityFlags.NOTHING: drop["reprodata"]["rg_parenthashes"] = {} return drop
def accumulate_pg_drop_data(drop: dict): """ Accumulate relevant reproducibility fields for a single drop at the physical graph level. :param drop: :return: A dictionary containing accumulated reproducibility data for a given drop. """ rmode = rflag_caster(drop['reprodata']['rmode']) if not rmode_supported(rmode): logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT drop['reprodata']['rmode'] = str(rmode.value) data = {} if rmode == ReproducibilityFlags.REPLICATE_COMP or rmode == ReproducibilityFlags.RECOMPUTE: data['node'] = drop['node'] data['island'] = drop['island'] return data
def init_lg_repro_drop_data(drop: dict): """ Creates and appends per-drop reproducibility information at the logical graph stage. :param drop: :return: The same drop with appended reproducibility information """ rmode = rflag_caster(drop['reprodata']['rmode']) if not rmode_supported(rmode): logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT drop['reprodata']['rmode'] = str(rmode.value) data = accumulate_lg_drop_data(drop, rmode) merkletree = MerkleTree(data.items(), common_hash) data['merkleroot'] = merkletree.merkle_root drop['reprodata']['lg_data'] = data drop['reprodata']['lg_parenthashes'] = {} return drop
def accumulate_pgt_partition_drop_data(drop: dict): """ Is as combination of unroll drop data :param drop: :return: """ rmode = rflag_caster(drop['reprodata']['rmode']) if not rmode_supported(rmode): logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT drop['reprodata']['rmode'] = str(rmode.value) data = accumulate_pgt_unroll_drop_data(drop) # This is the only piece of new information added at the partition level # It is only pertinent to Repetition and Computational replication if rmode == ReproducibilityFlags.REPLICATE_COMP or rmode == ReproducibilityFlags.RECOMPUTE: data['node'] = drop['node'][1:] data['island'] = drop['island'][1:] return data
def init_runtime_repro_data(rg: dict, reprodata: dict): """ Adds reproducibility data at the runtime level to graph-wide values. :param rg: :param reprodata: :return: """ rmode = rflag_caster(reprodata['rmode']) if not rmode_supported(rmode): # TODO: Logging needs sessionID at this stage # logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT reprodata['rmode'] = str(rmode.value) for drop_id, drop in rg.items(): init_rg_repro_drop_data(drop) leaves, visited = build_blockdag(list(rg.values()), 'rg') reprodata['signature'] = agglomerate_leaves(leaves) rg['reprodata'] = reprodata # logger.info("Reproducibility data finished at runtime level") return rg
def init_pg_repro_data(pg: list): """ Handles adding reproducibility data at the physical graph template level. :param pg: The logical graph data structure (a list of drops + reprodata dictionary) :return: The same pg object with new information appended """ reprodata = pg.pop() rmode = rflag_caster(reprodata['rmode']) if not rmode_supported(rmode): logger.warning("Requested reproducibility mode %s not yet implemented", str(rmode)) rmode = REPRO_DEFAULT reprodata['rmode'] = str(rmode.value) for drop in pg: init_pg_repro_drop_data(drop) leaves, visited = build_blockdag(pg, 'pg') reprodata['signature'] = agglomerate_leaves(leaves) pg.append(reprodata) logger.info("Reproducibility data finished at PG level") return pg
def init_pg_repro_drop_data(drop: dict): """ Creates and appends per-drop reproducibility information at the physical graph stage. :param drop: The drop description :return: The same drop with appended reproducibility information """ level = rflag_caster(drop["reprodata"]["rmode"]) data = accumulate_pg_drop_data(drop) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: merkletree = MerkleTree(data[rmode.name].items(), common_hash) data[rmode.name]["merkleroot"] = merkletree.merkle_root drop["reprodata"][rmode.name]["pg_parenthashes"] = {} drop["reprodata"][rmode.name]["pg_data"] = data[rmode.name] else: merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root # Separated so chaining can happen on independent elements (or both later) drop["reprodata"]["pg_parenthashes"] = {} drop["reprodata"]["pg_data"] = data return drop
def accumulate_pg_drop_data(drop: dict): """ Accumulate relevant reproducibility fields for a single drop at the physical graph level. :param drop: :return: A dictionary containing accumulated reproducibility data for a given drop. """ level = rflag_caster(drop["reprodata"]["rmode"]) if not rmode_supported(level): logger.warning("Requested reproducibility mode %s not yet implemented", str(level)) level = REPRO_DEFAULT drop["reprodata"]["rmode"] = str(level.value) if level == ReproducibilityFlags.ALL: data = {} for rmode in ALL_RMODES: pg_fields = pg_block_fields(rmode) data[rmode.name] = extract_fields(drop, pg_fields) else: pg_fields = pg_block_fields(level) data = extract_fields(drop, pg_fields) return data
def append_pgt_repro_data(drop: dict, data: dict): """ Adds provided data dictionary to drop description at PGT level. :param drop: The drop description :param data: The data to be added - arbitrary dictionary :return: """ level = rflag_caster(drop["reprodata"]["rmode"]) if level == ReproducibilityFlags.ALL: for rmode in ALL_RMODES: merkletree = MerkleTree(data[rmode.name].items(), common_hash) data[rmode.name]["merkleroot"] = merkletree.merkle_root drop["reprodata"][rmode.name]["pgt_parenthashes"] = {} drop["reprodata"][rmode.name]["pgt_data"] = data[rmode.name] else: merkletree = MerkleTree(data.items(), common_hash) data["merkleroot"] = merkletree.merkle_root # Separated so chaining can happen on independent elements (or both later) drop["reprodata"]["pgt_parenthashes"] = {} drop["reprodata"]["pgt_data"] = data return drop
def process_single(data): """ Processes reprodata containing a single signature. Builds a small dictionary mapping the 'rmode' to the signature """ return {rflag_caster(data.get("rmode")).value: data.get("signature")}
def lg_build_blockdag(logical_graph: dict, level=None): """ Uses Kahn's algorithm to topologically sort a logical graph dictionary. Exploits that a DAG contains at least one node with in-degree 0. Processes drops in-order. O(V + E) time complexity. :param logical_graph: The logical graph description (template or actual) :return: leaves set and the list of visited components (in order). """ dropset = {} # Also contains in-degree information neighbourset = {} roots = [] leaves = [] visited = [] queue = collections.deque() # TODO: Deal with MKN/Scatter Input drops for drop in logical_graph.get("nodeDataArray", []): did = int(drop["key"]) dropset[did] = [drop, 0, 0] neighbourset[did] = [] for edge in logical_graph.get("linkDataArray", []): src = int(edge["from"]) dest = int(edge["to"]) dropset[dest][1] += 1 dropset[src][2] += 1 neighbourset[src].append(dest) # did == 'drop id' for did, drop in dropset.items(): if drop[1] == 0: queue.append(did) roots.append(did) if not neighbourset[did]: # Leaf node leaves.append(did) while queue: did = queue.pop() # Process if "reprodata" not in dropset[did][0]: continue build_lg_block_data(dropset[did][0], level) visited.append(did) rmode = rflag_caster(dropset[did][0]["reprodata"]["rmode"]) if rmode == ReproducibilityFlags.ALL: rmode = level # Only building one layer at a time. for neighbour in neighbourset[did]: dropset[neighbour][1] -= 1 parenthash = {} if rmode != ReproducibilityFlags.NOTHING: if rmode == ReproducibilityFlags.REPRODUCE: if (dropset[did][0]["category"] in STORAGE_TYPES and (dropset[did][1] == 0 or dropset[did][2] == 0) and (did in roots or did in leaves)): # Add my new hash to the parent-hash list if did not in parenthash: if level is None: parenthash[did] = dropset[did][0]["reprodata"][ "lg_blockhash"] else: parenthash[did] = dropset[did][0]["reprodata"][ level.name]["lg_blockhash"] # parenthash.append(dropset[did][0]['reprodata']['lg_blockhash']) else: # Add my parenthashes to the parent-hash list if level is None: parenthash.update(dropset[did][0]["reprodata"] ["lg_parenthashes"]) else: parenthash.update(dropset[did][0]["reprodata"][ level.name]["lg_parenthashes"]) # parenthash.extend(dropset[did][0]['reprodata']['lg_parenthashes']) if rmode != ReproducibilityFlags.REPRODUCE: # Non-compressing behaviour if level is None: parenthash[did] = dropset[did][0]["reprodata"][ "lg_blockhash"] else: parenthash[did] = dropset[did][0]["reprodata"][ level.name]["lg_blockhash"] # parenthash.append(dropset[did][0]['reprodata']['lg_blockhash']) # Add our new hash to the parent-hash list # We deal with duplicates later if level is None: dropset[neighbour][0]["reprodata"][ "lg_parenthashes"].update(parenthash) else: dropset[neighbour][0]["reprodata"][ level.name]["lg_parenthashes"].update(parenthash) if dropset[neighbour][1] == 0: # Add drops at the DAG-frontier queue.append(neighbour) if len(visited) != len(dropset): logger.warning("Untraversed graph") logger.info("BlockDAG Generated at LG/T level") for i, leaf in enumerate(leaves): if level is None: # WARNING: Remove once dealt with MKN Nodes if "reprodata" in dropset[leaf][0]: leaves[i] = dropset[leaf][0]["reprodata"].get( "lg_blockhash", "") else: if "reprodata" in dropset[leaf][0]: leaves[i] = dropset[leaf][0]["reprodata"][level.name].get( "lg_blockhash", "") return leaves, visited
def lg_build_blockdag(lg: dict): """ Uses Kahn's algorithm to topologically sort a logical graph dictionary. Exploits that a DAG contains at least one node with in-degree 0. Processes drops in-order. O(V + E) time complexity. :param lg: :return: """ from collections import deque dropset = {} # Also contains in-degree information neighbourset = {} leaves = [] visited = [] q = deque() for drop in lg['nodeDataArray']: did = int(drop['key']) dropset[did] = [drop, 0, 0] neighbourset[did] = [] for edge in lg['linkDataArray']: src = int(edge['from']) dest = int(edge['to']) dropset[dest][1] += 1 dropset[src][2] += 1 neighbourset[src].append(dest) # did == 'drop id' for did in dropset: if dropset[did][1] == 0: q.append(did) if not neighbourset[did]: # Leaf node leaves.append(did) while q: did = q.pop() # Process build_lg_block_data(dropset[did][0]) visited.append(did) rmode = rflag_caster(dropset[did][0]['reprodata']['rmode']).value for n in neighbourset[did]: dropset[n][1] -= 1 parenthash = {} if rmode == ReproducibilityFlags.REPRODUCE.value: if dropset[did][0]['categoryType'] == Categories.DATA \ and (dropset[did][1] == 0 or dropset[did][2] == 0): # Add my new hash to the parent-hash list if did not in parenthash.keys(): parenthash[did] = dropset[did][0]['reprodata'][ 'lg_blockhash'] # parenthash.append(dropset[did][0]['reprodata']['lg_blockhash']) else: # Add my parenthashes to the parent-hash list parenthash.update( dropset[did][0]['reprodata']['lg_parenthashes']) # parenthash.extend(dropset[did][0]['reprodata']['lg_parenthashes']) if rmode != ReproducibilityFlags.REPRODUCE.value: # Non-compressing behaviour parenthash[did] = dropset[did][0]['reprodata']['lg_blockhash'] # parenthash.append(dropset[did][0]['reprodata']['lg_blockhash']) # Add our new hash to the parent-hash list dropset[n][0]['reprodata']['lg_parenthashes'].update( parenthash) # We deal with duplicates later if dropset[n][1] == 0: # Add drops at the DAG-frontier q.append(n) if len(visited) != len(dropset): raise Exception("Untraversed graph") logger.info("BlockDAG Generated at LG/T level") for i in range(len(leaves)): leaf = leaves[i] leaves[i] = dropset[leaf][0]['reprodata']['lg_blockhash'] return leaves, visited
def build_blockdag(drops: list, abstraction: str = "pgt", level=None): """ Uses Kahn's algorithm to topologically sort a logical graph dictionary. Exploits that a DAG contains at least one node with in-degree 0. Processes drops in-order. O(V + E) time complexity. :param drops: The list of drops :param abstraction: The level of graph abstraction 'pgt' || 'pg' :return: """ blockstr = "pgt" parentstr = "pgt_parenthashes" block_builder = build_pgt_block_data if abstraction == "pg": blockstr = "pg" parentstr = "pg_parenthashes" block_builder = build_pg_block_data if abstraction == "rg": blockstr = "rg" parentstr = "rg_parenthashes" block_builder = build_rg_block_data dropset = {} neighbourset = {} roots = [] leaves = [] visited = [] queue = collections.deque() for drop in drops: did = drop["oid"] dropset[did] = [drop, 0, 0] for drop in drops: did = drop["oid"] neighbourset[did] = [] if "outputs" in drop: # Assumes the model where all edges are defined from source to destination. # This may not always be the case. for dest in drop["outputs"]: if isinstance(dest, dict): dest = next(iter(dest)) dropset[dest][1] += 1 dropset[did][2] += 1 neighbourset[did].append(dest) if ("consumers" in drop ): # There may be some bizarre scenario when a drop has both for dest in drop["consumers"]: if isinstance(dest, dict): dest = next(iter(dest)) dropset[dest][1] += 1 dropset[did][2] += 1 neighbourset[did].append( dest) # TODO: Appending may not be correct behaviour for did, drop_val in dropset.items(): if drop_val[1] == 0: queue.append(did) roots.append(did) if not neighbourset[did]: # Leaf node leaves.append(did) while queue: did = queue.pop() block_builder(dropset[did][0], level) visited.append(did) rmode = rflag_caster(dropset[did][0]["reprodata"]["rmode"]) if rmode == ReproducibilityFlags.ALL: rmode = level for neighbour in neighbourset[did]: dropset[neighbour][1] -= 1 parenthash = {} if rmode != ReproducibilityFlags.NOTHING: if rmode == ReproducibilityFlags.REPRODUCE: # WARNING: Hack! may break later, proceed with caution if level is None: category = dropset[did][0]["reprodata"]["lgt_data"][ "category"] else: category = dropset[did][0]["reprodata"][ rmode.name]["lgt_data"]["category"] if (category in STORAGE_TYPES and (dropset[did][1] == 0 or dropset[did][2] == 0) and (did in roots or did in leaves)): # Add my new hash to the parent-hash list if did not in parenthash: if level is None: parenthash[did] = dropset[did][0]["reprodata"][ blockstr + "_blockhash"] else: parenthash[did] = dropset[did][0]["reprodata"][ level.name][blockstr + "_blockhash"] # parenthash.append(dropset[did][0]['reprodata'] \ # [blockstr + "_blockhash"]) else: # Add my parenthashes to the parent-hash list if level is None: parenthash.update( dropset[did][0]["reprodata"][parentstr]) else: parenthash.update(dropset[did][0]["reprodata"][ level.name][parentstr]) if rmode != ReproducibilityFlags.REPRODUCE: if level is None: parenthash[did] = dropset[did][0]["reprodata"][ blockstr + "_blockhash"] else: parenthash[did] = dropset[did][0]["reprodata"][ level.name][blockstr + "_blockhash"] # Add our new hash to the parent-hash list if on the critical path if rmode == ReproducibilityFlags.RERUN: if "iid" in dropset[did][0].keys(): if (dropset[did][0]["iid"] == "0/0" ): # TODO: This is probably wrong if level is None: dropset[neighbour][0]["reprodata"][ parentstr].update(parenthash) else: dropset[neighbour][0]["reprodata"][ level.name][parentstr].update(parenthash) else: if level is None: dropset[neighbour][0]["reprodata"][ parentstr].update(parenthash) else: dropset[neighbour][0]["reprodata"][ level.name][parentstr].update(parenthash) elif rmode != ReproducibilityFlags.RERUN: if level is None: dropset[neighbour][0]["reprodata"][parentstr].update( parenthash) else: dropset[neighbour][0]["reprodata"][ level.name][parentstr].update(parenthash) if dropset[neighbour][1] == 0: queue.append(neighbour) if len(visited) != len(dropset): logger.warning("Not a DAG") for i, leaf in enumerate(leaves): if level is None: leaves[i] = dropset[leaf][0]["reprodata"][blockstr + "_blockhash"] else: leaves[i] = dropset[leaf][0]["reprodata"][level.name][blockstr + "_blockhash"] return leaves, visited