def test_lg_scatter_rerun(self):
"""
Tests how rerunning treats such a graph.
Expected behaviour should be the same as any other type of graph - they are all logical
components
"""
lgt = _init_graph(
"test/reproducibility/reproGraphs/simpleScatter.graph")
init_lgt_repro_data(lgt, rmode=ReproducibilityFlags.RERUN.value)
init_lg_repro_data(lgt)
visited = lg_build_blockdag(lgt)[1]
scatter_drop = lgt["nodeDataArray"][1]
app_drop = lgt["nodeDataArray"][2]
scatter_inter_drop = lgt["nodeDataArray"][3]
# Checks that the input app drop is the parent of the main application
self.assertEqual(
list(app_drop["reprodata"]["lg_parenthashes"].values())[0],
scatter_inter_drop["reprodata"]["lg_blockhash"],
)
# Checks that the scatter drop is the parent of the input drop
self.assertEqual(
list(scatter_inter_drop["reprodata"]["lg_parenthashes"].values())
[0],
scatter_drop["reprodata"]["lg_blockhash"],
)
self.assertEqual(visited, [-1, -2, -5, -3, -6, -7, -9])
def test_lg_blockdag_empty(self):
"""
Tests an empty graph. Should fail gracefully.
"""
lgt = self.init_graph("topoGraphs/testEmpty.graph")
init_lgt_repro_data(lgt, "1")
init_lg_repro_data(lgt)
leaves, visited = lg_build_blockdag(lgt)
self.assertTrue(visited == [])
def test_lg_blockdag_single(self):
"""
Tests a single drop
A
"""
lgt = self.init_graph("topoGraphs/testSingle.graph")
init_lgt_repro_data(lgt, "1")
init_lg_repro_data(lgt)
leaves, visited = lg_build_blockdag(lgt)
self.assertTrue(visited == [-1])
def test_single(self):
"""
Tests a single drop
A
"""
lgt = _init_graph("topoGraphs/testSingle.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
leaves = lg_build_blockdag(lgt)[0]
self.assertTrue(len(leaves) == 1)
def test_twolines(self):
"""
A graph with two starts and two ends
A --> B
C --> D
"""
lgt = _init_graph("topoGraphs/testTwoLines.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
leaves = lg_build_blockdag(lgt)[0]
self.assertTrue(leaves[0] == leaves[1])
def test_lg_blockdag_twolines(self):
"""
A graph with two starts and two ends
A --> B
C --> D
"""
lgt = self.init_graph("topoGraphs/testTwoLines.graph")
init_lgt_repro_data(lgt, "1")
init_lg_repro_data(lgt)
leaves, visited = lg_build_blockdag(lgt)
self.assertTrue(visited == [-2, -3, -1, -4])
def test_twoend(self):
"""
A graph with two ends
--> B
A
--> C
"""
lgt = _init_graph("topoGraphs/testTwoEnd.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
leaves = lg_build_blockdag(lgt)[0]
self.assertFalse(leaves[0] == leaves[1])
def test_single(self):
"""
Tests a single drop
A
"""
# f = pkg_resources.resource_stream("test.reproducibility","topoGraphs/testSingle.graph")
# lgt = json.load(f)
lgt = _init_graph("topoGraphs/testSingle.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
leaves = lg_build_blockdag(lgt)[0]
self.assertTrue(len(leaves) == 1)
def test_computation_sandwich(self):
"""
Tests that an internal data drop surrounded by computing drops is handled correctly.
"""
lgt = _init_graph("topoGraphs/computationSandwich.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
lg_build_blockdag(lgt)
parenthashes = list(
lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values())
# Not going to get anything out of this, since reproduce only cares about terminal data.
self.assertEqual(0, len(parenthashes))
def test_lg_blockdag_twostart(self):
"""
A graph with two starts
A -->
C
B -->
"""
lgt = self.init_graph("topoGraphs/testTwoStart.graph")
init_lgt_repro_data(lgt, "1")
init_lg_repro_data(lgt)
leaves, visited = lg_build_blockdag(lgt)
self.assertTrue(visited == [-3, -1, -2])
def test_lg_blockdag_twoend(self):
"""
A graph with two ends
--> B
A
--> C
"""
lgt = self.init_graph("topoGraphs/testTwoEnd.graph")
init_lgt_repro_data(lgt, "1")
init_lg_repro_data(lgt)
leaves, visited = lg_build_blockdag(lgt)
self.assertTrue(visited == [-1, -3, -2])
def test_data_funnel(self):
"""
Tests that two data sources are collected in a single downstream data drop
"""
lgt = _init_graph("topoGraphs/dataFunnel.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
lg_build_blockdag(lgt)
sourcehash = lgt["nodeDataArray"][1]["reprodata"]["lg_blockhash"]
parenthashes = list(
lgt["nodeDataArray"][3]["reprodata"]["lg_parenthashes"].values())
self.assertTrue(sourcehash == parenthashes[0]
and len(parenthashes) == 1)
def test_computation_sandwich(self):
"""
Tests that an internal data drop surrounded by computing drops is handled correctly.
"""
lgt = _init_graph("topoGraphs/computationSandwich.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
lg_build_blockdag(lgt)
sourcehash = lgt["nodeDataArray"][1]["reprodata"]["lg_blockhash"]
parenthashes = list(
lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values())
self.assertTrue(sourcehash == parenthashes[0]
and len(parenthashes) == 1)
def test_data_sandwich(self):
"""
Tests two data drops with an interim computing drop
:return:
"""
lgt = _init_graph("topoGraphs/dataSandwich.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
lg_build_blockdag(lgt)
sourcehash = lgt["nodeDataArray"][0]["reprodata"]["lg_blockhash"]
parenthashes = list(
lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values())
self.assertTrue(sourcehash == parenthashes[0]
and len(parenthashes) == 1)
def test_data_fan(self):
"""
Tests that a single data source scatters its signature to downstream data drops.
"""
lgt = _init_graph("topoGraphs/dataFan.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
lg_build_blockdag(lgt)
sourcehash = lgt["nodeDataArray"][0]["reprodata"]["lg_blockhash"]
parenthash1 = list(
lgt["nodeDataArray"][2]["reprodata"]["lg_parenthashes"].values())
parenthash2 = list(
lgt["nodeDataArray"][3]["reprodata"]["lg_parenthashes"].values())
self.assertTrue(parenthash1 == parenthash2
and parenthash1[0] == sourcehash)
def jsonbody_post():
"""
Post graph JSON representation to LG or PG manager
"""
# see the table in http://bottlepy.org/docs/dev/tutorial.html#html-form-handling
lg_name = request.forms["lg_name"]
if lg_exists(lg_name):
rmode = request.forms.get("rmode", str(REPRO_DEFAULT.value))
lg_content = request.forms["lg_content"]
try:
lg_content = json.loads(lg_content)
except JSONDecodeError:
logger.warning(f"Could not decode lgt {lg_content}")
lg_content = init_lg_repro_data(init_lgt_repro_data(lg_content, rmode))
lg_path = "{0}/{1}".format(lg_dir, lg_name)
post_sem.acquire()
try:
# overwrite file on disks
# print "writing to {0}".format(lg_path)
with open(lg_path, "w") as f:
if isinstance(lg_content, dict):
lg_content = json.dumps(lg_content)
f.write(lg_content)
except Exception as excmd2:
response.status = 500
return "Fail to save logical graph {0}:{1}".format(
lg_name, str(excmd2))
finally:
post_sem.release()
else:
response.status = 404
return "{0}: logical graph {1} not found\n".format(err_prefix, lg_name)
def test_twostart(self):
"""
A graph with two starts
A -->
C
B -->
"""
lgt = _init_graph("topoGraphs/testTwoStart.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
leaves = lg_build_blockdag(lgt)[0]
parenthashes = list(
lgt["nodeDataArray"][1]["reprodata"]["lg_parenthashes"].values())
self.assertTrue(
len(leaves) == 1 and len(parenthashes) == 2
and parenthashes[0] == parenthashes[1])
def test_data_fan(self):
"""
Tests that a single data source scatters its signature to downstream data drops.
"""
lgt = _init_graph("topoGraphs/dataFan.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
for rmode in ALL_RMODES:
lg_build_blockdag(lgt, rmode)
sourcehash = lgt["nodeDataArray"][0]["reprodata"][
rmode.name]["lg_blockhash"]
parenthash1 = list(lgt["nodeDataArray"][2]["reprodata"][rmode.name]
["lg_parenthashes"].values())
parenthash2 = list(lgt["nodeDataArray"][3]["reprodata"][rmode.name]
["lg_parenthashes"].values())
self.assertTrue(parenthash1 == parenthash2)
if rmode != ReproducibilityFlags.REPRODUCE:
self.assertTrue(parenthash1[0] == sourcehash)
def test_data_funnel(self):
"""
Tests that two data sources are collected in a single downstream data drop
"""
lgt = _init_graph("topoGraphs/dataFunnel.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
for rmode in ALL_RMODES:
lg_build_blockdag(lgt, rmode)
sourcehash = lgt["nodeDataArray"][1]["reprodata"][
rmode.name]["lg_blockhash"]
parenthashes = list(lgt["nodeDataArray"][3]["reprodata"][
rmode.name]["lg_parenthashes"].values())
if rmode != ReproducibilityFlags.REPRODUCE:
self.assertTrue(sourcehash == parenthashes[0]
and len(parenthashes) == 1)
else:
self.assertTrue(len(parenthashes) == 2)
def test_computation_sandwich(self):
"""
Tests that an internal data drop surrounded by computing drops is handled correctly.
"""
lgt = _init_graph("topoGraphs/computationSandwich.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
for rmode in ALL_RMODES:
lg_build_blockdag(lgt, rmode)
sourcehash = lgt["nodeDataArray"][1]["reprodata"][
rmode.name]["lg_blockhash"]
parenthashes = list(lgt["nodeDataArray"][2]["reprodata"][
rmode.name]["lg_parenthashes"].values())
if rmode != ReproducibilityFlags.REPRODUCE:
self.assertTrue(sourcehash == parenthashes[0]
and len(parenthashes) == 1)
else:
self.assertTrue(len(parenthashes) == 0)
def test_data_sandwich(self):
"""
Tests two data drops with an interim computing drop
:return:
"""
lgt = _init_graph("topoGraphs/dataSandwich.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
for rmode in ALL_RMODES:
lg_build_blockdag(lgt, rmode)
sourcehash = lgt["nodeDataArray"][0]["reprodata"][
rmode.name]["lg_blockhash"]
parenthashes = list(lgt["nodeDataArray"][2]["reprodata"][
rmode.name]["lg_parenthashes"].values())
self.assertTrue(len(parenthashes) == 1)
if rmode != ReproducibilityFlags.REPRODUCE:
self.assertTrue(sourcehash == parenthashes[0])
else:
self.assertTrue(sourcehash != parenthashes[0])
def test_twostart(self):
"""
A graph with two starts
A -->
C
B -->
"""
lgt = _init_graph("topoGraphs/testTwoStart.graph")
init_lgt_repro_data(lgt, rmode=str(self.rmode.value))
init_lg_repro_data(lgt)
for rmode in ALL_RMODES:
leaves = lg_build_blockdag(lgt, rmode)[0]
parenthashes = list(lgt["nodeDataArray"][1]["reprodata"][
rmode.name]["lg_parenthashes"].values())
self.assertTrue(len(leaves) == 1)
if rmode != ReproducibilityFlags.REPRODUCE:
self.assertTrue(len(parenthashes) == 2)
self.assertTrue(parenthashes[0] == parenthashes[1])
else:
self.assertTrue(len(parenthashes) == 0)
def dlg_fill(parser, args):
tool.add_logging_options(parser)
_add_output_options(parser)
parser.add_option('-L',
'--logical-graph',
default='-',
help="Path to the Logical Graph (default: stdin)")
parser.add_option(
'-p',
'--parameter',
action='append',
help="Parameter specification (either 'name=value' or a JSON string)",
default=[])
parser.add_option(
'-R',
'--reproducibility',
default='0',
help="Level of reproducibility. Default 0 (NOTHING). Accepts '0'-'5'")
(opts, args) = parser.parse_args(args)
tool.setup_logging(opts)
dump = _setup_output(opts)
def param_spec_type(s):
if s.startswith('{'):
return 'json'
elif '=' in s:
return 'kv'
else:
return None
# putting all parameters together in a single dictionary
for p in opts.parameter:
if param_spec_type(p) is None:
parser.error(
'Parameter %s is neither JSON nor has it key=value form' % p)
params = [
p.split('=') for p in opts.parameter if param_spec_type(p) == 'kv'
]
params = dict(params)
for json_param in (json.loads(p) for p in opts.parameter
if param_spec_type(p) == 'json'):
params.update(json_param)
from ..dropmake.pg_generator import fill
graph = fill(_open_i(opts.logical_graph), params)
dump(init_lg_repro_data(init_lgt_repro_data(graph, opts.reproducibility)))
def test_lgt_init(self):
_graph = {}
sig = init_lgt_repro_data(_graph, "0")
self.assertNotIn("reprodata", sig)
def prepare_lgt(filename, rmode: str):
return init_lg_repro_data(init_lgt_repro_data(load_lg(filename), rmode))