def test_lg_scatter_rerun(self): """ Tests how rerunning treats such a graph. Expected behaviour should be the same as any other type of graph - they are all logical components """ lgt = _init_graph( "test/reproducibility/reproGraphs/simpleScatter.graph") init_lgt_repro_data(lgt, rmode=ReproducibilityFlags.RERUN.value) init_lg_repro_data(lgt) visited = lg_build_blockdag(lgt)[1] scatter_drop = lgt["nodeDataArray"][1] app_drop = lgt["nodeDataArray"][2] scatter_inter_drop = lgt["nodeDataArray"][3] # Checks that the input app drop is the parent of the main application self.assertEqual( list(app_drop["reprodata"]["lg_parenthashes"].values())[0], scatter_inter_drop["reprodata"]["lg_blockhash"], ) # Checks that the scatter drop is the parent of the input drop self.assertEqual( list(scatter_inter_drop["reprodata"]["lg_parenthashes"].values()) [0], scatter_drop["reprodata"]["lg_blockhash"], ) self.assertEqual(visited, [-1, -2, -5, -3, -6, -7, -9])
def test_lg_blockdag_empty(self): """ Tests an empty graph. Should fail gracefully. """ lgt = self.init_graph("topoGraphs/testEmpty.graph") init_lgt_repro_data(lgt, "1") init_lg_repro_data(lgt) leaves, visited = lg_build_blockdag(lgt) self.assertTrue(visited == [])
def test_lg_blockdag_single(self): """ Tests a single drop A """ lgt = self.init_graph("topoGraphs/testSingle.graph") init_lgt_repro_data(lgt, "1") init_lg_repro_data(lgt) leaves, visited = lg_build_blockdag(lgt) self.assertTrue(visited == [-1])
def test_single(self): """ Tests a single drop A """ lgt = _init_graph("topoGraphs/testSingle.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) leaves = lg_build_blockdag(lgt)[0] self.assertTrue(len(leaves) == 1)
def test_twolines(self): """ A graph with two starts and two ends A --> B C --> D """ lgt = _init_graph("topoGraphs/testTwoLines.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) leaves = lg_build_blockdag(lgt)[0] self.assertTrue(leaves[0] == leaves[1])
def test_lg_blockdag_twolines(self): """ A graph with two starts and two ends A --> B C --> D """ lgt = self.init_graph("topoGraphs/testTwoLines.graph") init_lgt_repro_data(lgt, "1") init_lg_repro_data(lgt) leaves, visited = lg_build_blockdag(lgt) self.assertTrue(visited == [-2, -3, -1, -4])
def test_twoend(self): """ A graph with two ends --> B A --> C """ lgt = _init_graph("topoGraphs/testTwoEnd.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) leaves = lg_build_blockdag(lgt)[0] self.assertFalse(leaves[0] == leaves[1])
def test_single(self): """ Tests a single drop A """ # f = pkg_resources.resource_stream("test.reproducibility","topoGraphs/testSingle.graph") # lgt = json.load(f) lgt = _init_graph("topoGraphs/testSingle.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) leaves = lg_build_blockdag(lgt)[0] self.assertTrue(len(leaves) == 1)
def test_computation_sandwich(self): """ Tests that an internal data drop surrounded by computing drops is handled correctly. """ lgt = _init_graph("topoGraphs/computationSandwich.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) lg_build_blockdag(lgt) parenthashes = list( lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values()) # Not going to get anything out of this, since reproduce only cares about terminal data. self.assertEqual(0, len(parenthashes))
def test_lg_blockdag_twostart(self): """ A graph with two starts A --> C B --> """ lgt = self.init_graph("topoGraphs/testTwoStart.graph") init_lgt_repro_data(lgt, "1") init_lg_repro_data(lgt) leaves, visited = lg_build_blockdag(lgt) self.assertTrue(visited == [-3, -1, -2])
def test_lg_blockdag_twoend(self): """ A graph with two ends --> B A --> C """ lgt = self.init_graph("topoGraphs/testTwoEnd.graph") init_lgt_repro_data(lgt, "1") init_lg_repro_data(lgt) leaves, visited = lg_build_blockdag(lgt) self.assertTrue(visited == [-1, -3, -2])
def test_data_funnel(self): """ Tests that two data sources are collected in a single downstream data drop """ lgt = _init_graph("topoGraphs/dataFunnel.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) lg_build_blockdag(lgt) sourcehash = lgt["nodeDataArray"][1]["reprodata"]["lg_blockhash"] parenthashes = list( lgt["nodeDataArray"][3]["reprodata"]["lg_parenthashes"].values()) self.assertTrue(sourcehash == parenthashes[0] and len(parenthashes) == 1)
def test_computation_sandwich(self): """ Tests that an internal data drop surrounded by computing drops is handled correctly. """ lgt = _init_graph("topoGraphs/computationSandwich.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) lg_build_blockdag(lgt) sourcehash = lgt["nodeDataArray"][1]["reprodata"]["lg_blockhash"] parenthashes = list( lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values()) self.assertTrue(sourcehash == parenthashes[0] and len(parenthashes) == 1)
def test_data_sandwich(self): """ Tests two data drops with an interim computing drop :return: """ lgt = _init_graph("topoGraphs/dataSandwich.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) lg_build_blockdag(lgt) sourcehash = lgt["nodeDataArray"][0]["reprodata"]["lg_blockhash"] parenthashes = list( lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values()) self.assertTrue(sourcehash == parenthashes[0] and len(parenthashes) == 1)
def test_data_fan(self): """ Tests that a single data source scatters its signature to downstream data drops. """ lgt = _init_graph("topoGraphs/dataFan.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) lg_build_blockdag(lgt) sourcehash = lgt["nodeDataArray"][0]["reprodata"]["lg_blockhash"] parenthash1 = list( lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values()) parenthash2 = list( lgt["nodeDataArray"][3]["reprodata"]["lg_parenthashes"].values()) self.assertTrue(parenthash1 == parenthash2 and parenthash1[0] == sourcehash)
def jsonbody_post(): """ Post graph JSON representation to LG or PG manager """ # see the table in http://bottlepy.org/docs/dev/tutorial.html#html-form-handling lg_name = request.forms["lg_name"] if lg_exists(lg_name): rmode = request.forms.get("rmode", str(REPRO_DEFAULT.value)) lg_content = request.forms["lg_content"] try: lg_content = json.loads(lg_content) except JSONDecodeError: logger.warning(f"Could not decode lgt {lg_content}") lg_content = init_lg_repro_data(init_lgt_repro_data(lg_content, rmode)) lg_path = "{0}/{1}".format(lg_dir, lg_name) post_sem.acquire() try: # overwrite file on disks # print "writing to {0}".format(lg_path) with open(lg_path, "w") as f: if isinstance(lg_content, dict): lg_content = json.dumps(lg_content) f.write(lg_content) except Exception as excmd2: response.status = 500 return "Fail to save logical graph {0}:{1}".format( lg_name, str(excmd2)) finally: post_sem.release() else: response.status = 404 return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)
def test_twostart(self): """ A graph with two starts A --> C B --> """ lgt = _init_graph("topoGraphs/testTwoStart.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) leaves = lg_build_blockdag(lgt)[0] parenthashes = list( lgt["nodeDataArray"][1]["reprodata"]["lg_parenthashes"].values()) self.assertTrue( len(leaves) == 1 and len(parenthashes) == 2 and parenthashes[0] == parenthashes[1])
def test_data_fan(self): """ Tests that a single data source scatters its signature to downstream data drops. """ lgt = _init_graph("topoGraphs/dataFan.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) for rmode in ALL_RMODES: lg_build_blockdag(lgt, rmode) sourcehash = lgt["nodeDataArray"][0]["reprodata"][ rmode.name]["lg_blockhash"] parenthash1 = list(lgt["nodeDataArray"][2]["reprodata"][rmode.name] ["lg_parenthashes"].values()) parenthash2 = list(lgt["nodeDataArray"][3]["reprodata"][rmode.name] ["lg_parenthashes"].values()) self.assertTrue(parenthash1 == parenthash2) if rmode != ReproducibilityFlags.REPRODUCE: self.assertTrue(parenthash1[0] == sourcehash)
def test_data_funnel(self): """ Tests that two data sources are collected in a single downstream data drop """ lgt = _init_graph("topoGraphs/dataFunnel.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) for rmode in ALL_RMODES: lg_build_blockdag(lgt, rmode) sourcehash = lgt["nodeDataArray"][1]["reprodata"][ rmode.name]["lg_blockhash"] parenthashes = list(lgt["nodeDataArray"][3]["reprodata"][ rmode.name]["lg_parenthashes"].values()) if rmode != ReproducibilityFlags.REPRODUCE: self.assertTrue(sourcehash == parenthashes[0] and len(parenthashes) == 1) else: self.assertTrue(len(parenthashes) == 2)
def test_computation_sandwich(self): """ Tests that an internal data drop surrounded by computing drops is handled correctly. """ lgt = _init_graph("topoGraphs/computationSandwich.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) for rmode in ALL_RMODES: lg_build_blockdag(lgt, rmode) sourcehash = lgt["nodeDataArray"][1]["reprodata"][ rmode.name]["lg_blockhash"] parenthashes = list(lgt["nodeDataArray"][2]["reprodata"][ rmode.name]["lg_parenthashes"].values()) if rmode != ReproducibilityFlags.REPRODUCE: self.assertTrue(sourcehash == parenthashes[0] and len(parenthashes) == 1) else: self.assertTrue(len(parenthashes) == 0)
def test_data_sandwich(self): """ Tests two data drops with an interim computing drop :return: """ lgt = _init_graph("topoGraphs/dataSandwich.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) for rmode in ALL_RMODES: lg_build_blockdag(lgt, rmode) sourcehash = lgt["nodeDataArray"][0]["reprodata"][ rmode.name]["lg_blockhash"] parenthashes = list(lgt["nodeDataArray"][2]["reprodata"][ rmode.name]["lg_parenthashes"].values()) self.assertTrue(len(parenthashes) == 1) if rmode != ReproducibilityFlags.REPRODUCE: self.assertTrue(sourcehash == parenthashes[0]) else: self.assertTrue(sourcehash != parenthashes[0])
def test_twostart(self): """ A graph with two starts A --> C B --> """ lgt = _init_graph("topoGraphs/testTwoStart.graph") init_lgt_repro_data(lgt, rmode=str(self.rmode.value)) init_lg_repro_data(lgt) for rmode in ALL_RMODES: leaves = lg_build_blockdag(lgt, rmode)[0] parenthashes = list(lgt["nodeDataArray"][1]["reprodata"][ rmode.name]["lg_parenthashes"].values()) self.assertTrue(len(leaves) == 1) if rmode != ReproducibilityFlags.REPRODUCE: self.assertTrue(len(parenthashes) == 2) self.assertTrue(parenthashes[0] == parenthashes[1]) else: self.assertTrue(len(parenthashes) == 0)
def dlg_fill(parser, args): tool.add_logging_options(parser) _add_output_options(parser) parser.add_option('-L', '--logical-graph', default='-', help="Path to the Logical Graph (default: stdin)") parser.add_option( '-p', '--parameter', action='append', help="Parameter specification (either 'name=value' or a JSON string)", default=[]) parser.add_option( '-R', '--reproducibility', default='0', help="Level of reproducibility. Default 0 (NOTHING). Accepts '0'-'5'") (opts, args) = parser.parse_args(args) tool.setup_logging(opts) dump = _setup_output(opts) def param_spec_type(s): if s.startswith('{'): return 'json' elif '=' in s: return 'kv' else: return None # putting all parameters together in a single dictionary for p in opts.parameter: if param_spec_type(p) is None: parser.error( 'Parameter %s is neither JSON nor has it key=value form' % p) params = [ p.split('=') for p in opts.parameter if param_spec_type(p) == 'kv' ] params = dict(params) for json_param in (json.loads(p) for p in opts.parameter if param_spec_type(p) == 'json'): params.update(json_param) from ..dropmake.pg_generator import fill graph = fill(_open_i(opts.logical_graph), params) dump(init_lg_repro_data(init_lgt_repro_data(graph, opts.reproducibility)))
def test_lg_init(self): _graph = {} sig = init_lg_repro_data(_graph) self.assertNotIn("reprodata", sig)
def prepare_lgt(filename, rmode: str): return init_lg_repro_data(init_lgt_repro_data(load_lg(filename), rmode))