Python NullDROP.reproducibility_levelの例

コード例 #1

ファイル: test_drop.py プロジェクト: ICRAR/daliuge

 def test_drop_reproduce(self):
     a = NullDROP("a", "a")
     a.reproducibility_level = ReproducibilityFlags.REPRODUCE
     a.setCompleted()
     self.assertEqual(a.generate_merkle_data(), {})
     self.assertIsNone(a.merkleroot)
     pass

コード例 #2

ファイル: test_drop.py プロジェクト: ICRAR/daliuge

 def test_drop_replicate_comp(self):
     a = NullDROP("a", "a")
     a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
     a.setCompleted()
     self.assertEqual(a.generate_rerun_data(), {"status": DROPStates.COMPLETED})
     self.assertIsNotNone(a.merkleroot)
     pass

コード例 #3

ファイル: test_bash.py プロジェクト: pritchardn/daliuge

    def test_reproducibility(self):
        from dlg.common.reproducibility.constants import ReproducibilityFlags
        from dlg.drop import NullDROP
        a = BashShellApp('a', 'a', command="echo 'Hello world'")
        a.reproducibility_level = ReproducibilityFlags.RERUN
        a.setCompleted()
        b = NullDROP('b', 'b')
        b.reproducibility_level = ReproducibilityFlags.RERUN
        b.setCompleted()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPEAT
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'command': "echo 'Hello world'"})

        a.reproducibility_level = ReproducibilityFlags.REPRODUCE
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
        self.assertEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_rerun_data())

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_recompute_data())

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
        self.assertEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_repeat_data())

コード例 #4

ファイル: test_drop.py プロジェクト: ICRAR/daliuge

 def test_drop_recompute(self):
     a = NullDROP("a", "a")
     a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
     a.setCompleted()
     self.assertEqual(a.generate_merkle_data(), {"status": DROPStates.COMPLETED})
     self.assertIsNotNone(a.merkleroot)
     pass

コード例 #5

    def test_memory_reproducibility(self):
        from dlg.common.reproducibility.reproducibility import common_hash
        data = b'Helloworld'
        data_hash = common_hash(data)
        a = InMemoryDROP('a', 'a')
        a.write(data)
        a.reproducibility_level = ReproducibilityFlags.RERUN
        a.setCompleted()
        b = NullDROP('b', 'b')
        b.reproducibility_level = ReproducibilityFlags.RERUN
        b.setCompleted()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPEAT
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPRODUCE
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'data_hash': data_hash, 'status': DROPStates.COMPLETED})

コード例 #6

 def test_drop_replicate_total(self):
     a = NullDROP('a', 'a')
     a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
     a.setCompleted()
     self.assertEqual(a.generate_rerun_data(), {'status': DROPStates.COMPLETED})
     self.assertIsNotNone(a.merkleroot)
     pass

コード例 #7

 def test_drop_repeat(self):
     a = NullDROP('a', 'a')
     a.reproducibility_level = ReproducibilityFlags.REPEAT
     a.setCompleted()
     self.assertEqual(a.generate_merkle_data(), {'status': DROPStates.COMPLETED})
     self.assertIsNotNone(a.merkleroot)
     pass

コード例 #8

ファイル: test_pyfunc.py プロジェクト: pritchardn/daliuge

    def test_reproducibility(self):
        from dlg.common.reproducibility.constants import ReproducibilityFlags
        from dlg.drop import NullDROP
        a = _PyFuncApp('a', 'a', 'func3')
        a.run()
        b = NullDROP('b', 'b')
        a.reproducibility_level = ReproducibilityFlags.RERUN
        b.reproducibility_level = ReproducibilityFlags.RERUN
        a.setCompleted()
        b.setCompleted()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPEAT
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {'args': [], 'kwargs': {}})

        a.reproducibility_level = ReproducibilityFlags.REPRODUCE
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
        self.assertEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_rerun_data())

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_recompute_data())

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
        self.assertEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), a.generate_repeat_data())

コード例 #9

ファイル: test_drop.py プロジェクト: ICRAR/daliuge

    def test_memory_reproducibility(self):
        from dlg.common.reproducibility.reproducibility import common_hash

        data = b"Helloworld"
        data_hash = common_hash(data)
        a = InMemoryDROP("a", "a")
        a.write(data)
        a.reproducibility_level = ReproducibilityFlags.RERUN
        a.setCompleted()
        b = NullDROP("b", "b")
        b.reproducibility_level = ReproducibilityFlags.RERUN
        b.setCompleted()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPEAT
        a.commit()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
        a.commit()
        self.assertEqual(a.merkleroot, b.merkleroot)

        a.reproducibility_level = ReproducibilityFlags.REPRODUCE
        a.commit()
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(a.generate_merkle_data(), {"data_hash": data_hash})

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
        a.commit()
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(
            a.generate_merkle_data(),
            {"data_hash": data_hash, "status": DROPStates.COMPLETED},
        )

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
        a.commit()
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(
            a.generate_merkle_data(),
            {"data_hash": data_hash, "status": DROPStates.COMPLETED},
        )

        a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
        a.commit()
        self.assertNotEqual(a.merkleroot, b.merkleroot)
        self.assertEqual(
            a.generate_merkle_data(),
            {"data_hash": data_hash, "status": DROPStates.COMPLETED},
        )

コード例 #10

    def test_rdbms_reproducibility(self):
        dbfile = 'test_rdbms_drop.db'
        if os.path.isfile(dbfile):
            os.unlink(dbfile)

        b = NullDROP('b', 'b')
        b.reproducibility_level = ReproducibilityFlags.RERUN
        b.setCompleted()

        with contextlib.closing(sqlite3.connect(dbfile)) as conn:  # @UndefinedVariable
            with contextlib.closing(conn.cursor()) as cur:
                cur.execute('CREATE TABLE super_mega_table(a_string varchar(64) PRIMARY KEY, an_integer integer)');

        try:
            a = RDBMSDrop('a', 'a', dbmodule='sqlite3', dbtable='super_mega_table', dbparams={'database': dbfile})
            a.insert({'a_string': 'hello', 'an_integer': 0})
            a.insert({'a_string': 'hello1', 'an_integer': 1})
            a.reproducibility_level = ReproducibilityFlags.RERUN
            a.setCompleted()
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPEAT
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPRODUCE
            self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog})
            self.assertNotEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(a.generate_merkle_data(), {'query_log': a._querylog, 'status': DROPStates.COMPLETED})
        finally:
            os.unlink(dbfile)

コード例 #11

ファイル: test_drop.py プロジェクト: ICRAR/daliuge

    def test_rdbms_reproducibility(self):
        dbfile = "test_rdbms_drop.db"
        if os.path.isfile(dbfile):
            os.unlink(dbfile)

        b = NullDROP("b", "b")
        b.reproducibility_level = ReproducibilityFlags.RERUN
        b.setCompleted()

        with contextlib.closing(sqlite3.connect(dbfile)) as conn:  # @UndefinedVariable
            with contextlib.closing(conn.cursor()) as cur:
                cur.execute(
                    "CREATE TABLE super_mega_table(a_string varchar(64) PRIMARY KEY, an_integer integer)"
                )

        try:
            a = RDBMSDrop(
                "a",
                "a",
                dbmodule="sqlite3",
                dbtable="super_mega_table",
                dbparams={"database": dbfile},
            )
            a.insert({"a_string": "hello", "an_integer": 0})
            a.insert({"a_string": "hello1", "an_integer": 1})
            a.reproducibility_level = ReproducibilityFlags.RERUN
            a.setCompleted()
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPEAT
            a.commit()
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.RECOMPUTE
            a.commit()
            self.assertEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPRODUCE
            a.commit()
            self.assertEqual(a.generate_merkle_data(), {"query_log": a._querylog})
            self.assertNotEqual(a.merkleroot, b.merkleroot)

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_SCI
            a.commit()
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(
                a.generate_merkle_data(),
                {"query_log": a._querylog, "status": DROPStates.COMPLETED},
            )

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_COMP
            a.commit()
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(
                a.generate_merkle_data(),
                {"query_log": a._querylog, "status": DROPStates.COMPLETED},
            )

            a.reproducibility_level = ReproducibilityFlags.REPLICATE_TOTAL
            a.commit()
            self.assertNotEqual(a.merkleroot, b.merkleroot)
            self.assertEqual(
                a.generate_merkle_data(),
                {"query_log": a._querylog, "status": DROPStates.COMPLETED},
            )
        finally:
            os.unlink(dbfile)