コード例 #1
0
    def test_builder_checkup(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size, debug=True)
        generated = bucket.build(self.store_path)

        # make sure all files are there
        self.assertTrue(bucket.checkup(self.store_path))

        # remove one of the files
        remove_file = random.choice(list(generated.keys()))
        os.remove(os.path.join(self.store_path, remove_file))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))
コード例 #2
0
    def test_builder_checkup(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size)
        generated = bucket.build(self.store_path)

        # make sure all files are there
        self.assertTrue(bucket.checkup(self.store_path))

        # remove one of the files
        remove_file = random.choice(list(generated.keys()))
        os.remove(os.path.join(self.store_path, remove_file))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))
コード例 #3
0
ファイル: test_builder.py プロジェクト: F483/dataserv-client
    def test_builder_skips_existing(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"],
                         my_shard_size,
                         my_max_size,
                         debug=True)
        generated = bucket.build(self.store_path)

        timestamp = time.time()

        # remove one of the files
        remove_file = random.choice(list(generated.keys()))
        os.remove(os.path.join(self.store_path, remove_file))

        # generate only missing shard for testing
        bucket = Builder(addresses["epsilon"],
                         my_shard_size,
                         my_max_size,
                         debug=True)
        generated = bucket.build(self.store_path)

        # verify last access times
        for seed, file_hash in generated.items():
            path = os.path.join(self.store_path, seed)
            last_access = os.path.getmtime(path)
            if seed == remove_file:
                self.assertTrue(last_access > timestamp)
            else:
                self.assertTrue(last_access < timestamp)

        # make sure all files are there
        self.assertTrue(bucket.checkup(self.store_path))
コード例 #4
0
    def test_builder_skips_existing(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size)
        generated = bucket.build(self.store_path, False, False)

        timestamp = time.time()

        # remove one of the files
        remove_file = random.choice(list(generated.keys()))
        os.remove(os.path.join(self.store_path, remove_file))

        # generate only missing shard for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size)
        generated = bucket.build(self.store_path, False, False)

        # verify last access times
        for seed, file_hash in generated.items():
            path = os.path.join(self.store_path, seed)
            last_access = os.path.getmtime(path)
            if seed == remove_file:
                self.assertTrue(last_access > timestamp)
            else:
                self.assertTrue(last_access < timestamp)

        # make sure all files are there
        self.assertTrue(bucket.checkup(self.store_path))
コード例 #5
0
    def test_build_rebuild(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size)
        bucket.build(self.store_path)

        # remove one of the files
        r = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        os.remove(os.path.join(self.store_path, r))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))

        # rebuild
        bucket.build(self.store_path, rebuild=True)

        # check again, should pass
        self.assertTrue(bucket.checkup(self.store_path))
コード例 #6
0
    def test_build_rebuild(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size, debug=True)
        bucket.build(self.store_path)

        # remove one of the files
        remove_file = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        os.remove(os.path.join(self.store_path, remove_file))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))

        # rebuild
        bucket.build(self.store_path, rebuild=True)

        # check again, should pass
        self.assertTrue(bucket.checkup(self.store_path))
コード例 #7
0
 def test_use_folder_tree_clean(self):
     bucket = Builder(addresses["beta"], my_shard_size, my_max_size,
                      use_folder_tree=True)
     bucket.build(self.store_path)
     self.assertTrue(bucket.checkup(self.store_path))
     bucket.clean(self.store_path)
     def callback(a, d, files):
         self.assertTrue(len(files) == 0)
     os.walk(self.store_path, callback, None)
コード例 #8
0
ファイル: test_builder.py プロジェクト: kgdb/dataserv-client
    def test_use_folder_tree_clean(self):
        bucket = Builder(addresses["beta"], my_shard_size, my_max_size,
                         my_min_free_size,
                         use_folder_tree=True)
        bucket.build(self.store_path)
        self.assertTrue(bucket.checkup(self.store_path))
        bucket.clean(self.store_path)

        def callback(a, d, files):
            self.assertTrue(len(files) == 0)
        os.walk(self.store_path, callback, None)
コード例 #9
0
    def test_build_repair(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size,
                         my_min_free_size)
        bucket.build(self.store_path)

        # remove one of the files
        r = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        os.remove(os.path.join(self.store_path, r))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))

        # repair
        bucket.build(self.store_path, repair=True)

        # check again, should pass
        self.assertTrue(bucket.checkup(self.store_path))

        # modify one of the files
        o = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        path = os.path.join(self.store_path, o)
        sha256_org_file = partialhash.compute(path)

        # write some data
        with open(path, "a") as f:
            f.write("bad data is bad\n")

        # check their hashes
        sha256_mod_file = partialhash.compute(path)
        self.assertNotEqual(sha256_org_file, sha256_mod_file)

        # build without a repair should fail
        bucket.build(self.store_path)
        sha256_mod_file = partialhash.compute(path)
        self.assertNotEqual(sha256_org_file, sha256_mod_file)

        # build with a repair should pass
        bucket.build(self.store_path, repair=True)
        sha256_mod_file = partialhash.compute(path)
        self.assertEqual(sha256_org_file, sha256_mod_file)
コード例 #10
0
    def test_build_repair(self):
        # generate shards for testing
        bucket = Builder(addresses["epsilon"], my_shard_size, my_max_size,
                         my_min_free_size)
        bucket.build(self.store_path)

        # remove one of the files
        r = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        os.remove(os.path.join(self.store_path, r))

        # check again, should fail
        self.assertFalse(bucket.checkup(self.store_path))

        # repair
        bucket.build(self.store_path, repair=True)

        # check again, should pass
        self.assertTrue(bucket.checkup(self.store_path))

        # modify one of the files
        o = 'baf428097fa601fac185750483fd532abb0e43f9f049398290fac2c049cc2a60'
        path = os.path.join(self.store_path, o)
        sha256_org_file = partialhash.compute(path)

        # write some data
        with open(path, "a") as f:
            f.write("bad data is bad\n")

        # check their hashes
        sha256_mod_file = partialhash.compute(path)
        self.assertNotEqual(sha256_org_file, sha256_mod_file)

        # build without a repair should fail
        bucket.build(self.store_path)
        sha256_mod_file = partialhash.compute(path)
        self.assertNotEqual(sha256_org_file, sha256_mod_file)

        # build with a repair should pass
        bucket.build(self.store_path, repair=True)
        sha256_mod_file = partialhash.compute(path)
        self.assertEqual(sha256_org_file, sha256_mod_file)