def test_memory_reproducibility(self):
from dlg.common.reproducibility.reproducibility import common_hash
data = b'Helloworld'
data_hash = common_hash(data)
a = InMemoryDROP('a', 'a')
a.write(data)
a.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
b = NullDROP('b', 'b')
b.reproducibility_level = ReproducibilityFlags.RERUN
b.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})
def test_reproducibility(self):
from dlg.common.reproducibility.constants import ReproducibilityFlags
from dlg.drop import NullDROP
a = BashShellApp('a', 'a', command="echo 'Hello world'")
a.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
b = NullDROP('b', 'b')
b.reproducibility_level = ReproducibilityFlags.RERUN
b.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'command': "echo 'Hello world'"})
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
self.assertEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_rerun_data())
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_recompute_data())
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
self.assertEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_repeat_data())
def test_reproducibility(self):
from dlg.common.reproducibility.constants import ReproducibilityFlags
from dlg.drop import NullDROP
a = _PyFuncApp('a', 'a', 'func3')
a.run()
b = NullDROP('b', 'b')
a.reproducibility_level = ReproducibilityFlags.RERUN
b.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
b.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'args': [], 'kwargs': {}})
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
self.assertEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_rerun_data())
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_recompute_data())
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
self.assertEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), a.generate_repeat_data())
def test_sleepapp(self):
# Nothing fancy, just run it and be done with it
a = NullDROP("a", "a")
b = SleepApp("b", "b")
c = NullDROP("c", "c")
b.addInput(a)
b.addOutput(c)
self._test_graph_runs((a, b, c), a, c)
def test_sleepapp(self):
# Nothing fancy, just run it and be done with it
a = NullDROP('a', 'a')
b = SleepApp('b', 'b')
c = NullDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
a = NullDROP('a', 'a')
def test_memory_reproducibility(self):
from dlg.common.reproducibility.reproducibility import common_hash
data = b"Helloworld"
data_hash = common_hash(data)
a = InMemoryDROP("a", "a")
a.write(data)
a.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
b = NullDROP("b", "b")
b.reproducibility_level = ReproducibilityFlags.RERUN
b.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
a.commit()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
a.commit()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {"data_hash": data_hash})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"data_hash": data_hash, "status": DROPStates.COMPLETED},
)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"data_hash": data_hash, "status": DROPStates.COMPLETED},
)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"data_hash": data_hash, "status": DROPStates.COMPLETED},
)
def test_rdbms_reproducibility(self):
dbfile = 'test_rdbms_drop.db'
if os.path.isfile(dbfile):
os.unlink(dbfile)
b = NullDROP('b', 'b')
b.reproducibility_level = ReproducibilityFlags.RERUN
b.setCompleted()
with contextlib.closing(sqlite3.connect(dbfile)) as conn: # @UndefinedVariable
with contextlib.closing(conn.cursor()) as cur:
cur.execute('CREATE TABLE super_mega_table(a_string varchar(64) PRIMARY KEY, an_integer integer)');
try:
a = RDBMSDrop('a', 'a', dbmodule='sqlite3', dbtable='super_mega_table', dbparams={'database': dbfile})
a.insert({'a_string': 'hello', 'an_integer': 0})
a.insert({'a_string': 'hello1', 'an_integer': 1})
a.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog})
self.assertNotEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})
finally:
os.unlink(dbfile)
def test_randomarrayapp(self):
i = NullDROP("i", "i")
c = RandomArrayApp("c", "c", keep_array=True)
o = InMemoryDROP("o", "o")
c.addInput(i)
c.addOutput(o)
self._test_graph_runs((i, c, o), i, o)
marray = c._getArray()
data = pickle.loads(droputils.allDropContents(o))
v = marray == data
self.assertEqual(v.all(), True)
def test_drop_replicate_comp(self):
a = NullDROP("a", "a")
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
a.setCompleted()
self.assertEqual(a.generate_rerun_data(), {"status": DROPStates.COMPLETED})
self.assertIsNotNone(a.merkleroot)
pass
def test_drop_reproduce(self):
a = NullDROP("a", "a")
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
a.setCompleted()
self.assertEqual(a.generate_merkle_data(), {})
self.assertIsNone(a.merkleroot)
pass
def test_drop_recompute(self):
a = NullDROP("a", "a")
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
a.setCompleted()
self.assertEqual(a.generate_merkle_data(), {"status": DROPStates.COMPLETED})
self.assertIsNotNone(a.merkleroot)
pass
def test_drop_replicate_total(self):
a = NullDROP('a', 'a')
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
a.setCompleted()
self.assertEqual(a.generate_rerun_data(), {'status': DROPStates.COMPLETED})
self.assertIsNotNone(a.merkleroot)
pass
def test_drop_repeat(self):
a = NullDROP('a', 'a')
a.reproducibility_level = ReproducibilityFlags.REPEAT
a.setCompleted()
self.assertEqual(a.generate_merkle_data(), {'status': DROPStates.COMPLETED})
self.assertIsNotNone(a.merkleroot)
pass
def test_NullDROP(self):
"""
Check that the NullDROP is usable for testing
"""
a = NullDROP("A", "A", expectedSize=5)
a.write(b"1234")
a.write(b"5")
allContents = droputils.allDropContents(a)
self.assertFalse(allContents)
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir("/tmp")
dirname = "/tmp/.hidden"
dirname2 = "/tmp/.hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP("a", "a", dirname=dirname)
b = FileDROP("b", "b", dirname=dirname)
c = FileDROP("c", "c", dirname=dirname2)
d = FileDROP("d", "d", dirname=dirname2)
cont1 = DirectoryContainer("e", "e", dirname=dirname)
cont2 = DirectoryContainer("f", "f", dirname=dirname2)
# Paths are absolutely reported
self.assertEqual(os.path.realpath("/tmp/.hidden"), os.path.realpath(cont1.path))
self.assertEqual(
os.path.realpath("/tmp/.hidden/inside"), os.path.realpath(cont2.path)
)
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP("g", "g"))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP("h", "h"))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP("i", "i"))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
def test_rdbms_reproducibility(self):
dbfile = "test_rdbms_drop.db"
if os.path.isfile(dbfile):
os.unlink(dbfile)
b = NullDROP("b", "b")
b.reproducibility_level = ReproducibilityFlags.RERUN
b.setCompleted()
with contextlib.closing(sqlite3.connect(dbfile)) as conn: # @UndefinedVariable
with contextlib.closing(conn.cursor()) as cur:
cur.execute(
"CREATE TABLE super_mega_table(a_string varchar(64) PRIMARY KEY, an_integer integer)"
)
try:
a = RDBMSDrop(
"a",
"a",
dbmodule="sqlite3",
dbtable="super_mega_table",
dbparams={"database": dbfile},
)
a.insert({"a_string": "hello", "an_integer": 0})
a.insert({"a_string": "hello1", "an_integer": 1})
a.reproducibility_level = ReproducibilityFlags.RERUN
a.setCompleted()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPEAT
a.commit()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
a.commit()
self.assertEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPRODUCE
a.commit()
self.assertEqual(a.generate_merkle_data(), {"query_log": a._querylog})
self.assertNotEqual(a.merkleroot, b.merkleroot)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"query_log": a._querylog, "status": DROPStates.COMPLETED},
)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"query_log": a._querylog, "status": DROPStates.COMPLETED},
)
a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
a.commit()
self.assertNotEqual(a.merkleroot, b.merkleroot)
self.assertEqual(
a.generate_merkle_data(),
{"query_log": a._querylog, "status": DROPStates.COMPLETED},
)
finally:
os.unlink(dbfile)