def test_curr_src_num(self): # we can get/set current source number bucket = Bucket(self.workdir) self.assertEqual(bucket.get_current_source_num(), 0) bucket.set_current_source_num(12) self.assertEqual(bucket.get_current_source_num(), 12) return
def test_store_representation_string_key(self, cache_env): # we can store sources with their representations and a string key bucket = Bucket(str(cache_env.join("cache"))) res = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key="somekey") assert res == "1_1" assert ( cache_env / "cache" / "keys" / "1" / "1.key").read() == 'somekey'
def test_store_representation_string_key(self, cache_env): # we can store sources with their representations and a string key bucket = Bucket(str(cache_env.join("cache"))) res = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key="somekey") assert res == "1_1" assert (cache_env / "cache" / "keys" / "1" / "1.key").read() == 'somekey'
def test_get_representation_stored(self, cache_env): # we can get paths of representations bucket = Bucket(str(cache_env.join("cache"))) res1 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key=b'mykey') res2 = bucket.get_representation(res1) assert res1 == "1_1" assert res2 == cache_env / "cache" / "repr" / "1" / "1" / "result1.txt"
def test_get_representation_stored(self, cache_env): # we can get paths of representations bucket = Bucket(str(cache_env.join("cache"))) res1 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key=b'mykey') res2 = bucket.get_representation(res1) assert res1 == "1_1" assert res2 == cache_env / "cache" / "repr" / "1" / "1" / "result1.txt"
def test_store_representation_string_key(self): # we can store sources with their representations and a string key bucket = Bucket(self.workdir) res = bucket.store_representation( self.src_path1, self.result_path1, repr_key='somekey') exp_stored_key = os.path.join(self.workdir, b'keys', b'1', b'1.key') self.assertEqual( open(exp_stored_key, 'rb').read(), b'somekey') self.assertEqual(res, '1_1') return
def test_get_representation_stored(self): # we cannot get unstored representations bucket = Bucket(self.workdir) res1 = bucket.store_representation( self.src_path1, self.result_path1, repr_key=b'mykey') exp_repr_path = os.path.join( self.workdir, b'repr', b'1', b'1', b'resultfile1') res2 = bucket.get_representation(res1) self.assertEqual(res1, b'1_1') self.assertEqual(res2, exp_repr_path) return
def test_store_result_marker(self): bucket = Bucket(self.workdir) marker = bucket.storeResult( self.src_path1, self.result_path1, suffix='foo') self.assertEqual(marker, '1') marker = bucket.storeResult( self.src_path1, self.result_path2, suffix='foo') self.assertEqual(marker, '1') marker = bucket.storeResult( self.src_path2, self.result_path2, suffix='foo') self.assertEqual(marker, '2') return
def test_get_all_source_paths_ignore_nonsense(self): bucket = Bucket(self.workdir) open(os.path.join(self.workdir, 'sources', 'source3'), 'wb').write('Hi') open(os.path.join(self.workdir, 'sources', 'foo_3'), 'wb').write('Hi') paths1 = list(bucket.getAllSourcePaths()) bucket.storeResult( self.src_path1, self.result_path1, suffix='foo') paths2 = list(bucket.getAllSourcePaths()) self.assertEqual(paths1, []) self.assertEqual(len(paths2), 1)
def test_get_stored_source_num(self, cache_env): # we can test whether a source file is stored in a bucket already. bucket = Bucket(str(cache_env.join("cache"))) src1 = cache_env / "src1.txt" src2 = cache_env / "src2.txt" assert bucket.get_stored_source_num(str(src1)) is None assert bucket.get_stored_source_num(str(src2)) is None shutil.copyfile(str(src1), os.path.join(bucket.srcdir, "source_1")) assert bucket.get_stored_source_num(str(src1)) == 1 assert bucket.get_stored_source_num(str(src2)) is None shutil.copyfile(str(src2), os.path.join(bucket.srcdir, "source_2")) assert bucket.get_stored_source_num(str(src1)) == 1 assert bucket.get_stored_source_num(str(src2)) == 2
def test_get_source_path_ignore_nonsense(self): # Nonsense files in sourcedir are ignored. bucket = Bucket(self.workdir) open(os.path.join(self.workdir, 'sources', 'source3'), 'wb').write('Hi') open(os.path.join(self.workdir, 'sources', 'foo_3'), 'wb').write('Hi') path, marker = bucket.getSourcePath(self.src_path1) self.assertEqual((path, marker), (None, None)) bucket.storeResult(self.src_path1, self.result_path1) path, marker = bucket.getSourcePath(self.src_path1) self.assertEqual(marker, '1') return
def test_store_result_with_suffix(self): bucket = Bucket(self.workdir) bucket.storeResult(self.src_path1, self.result_path1, suffix='foo') files_are_equal = filecmp.cmp( self.src_path1, os.path.join(self.workdir, 'sources', 'source_1') ) self.assertTrue(files_are_equal, msg='source files differ') files_are_equal = filecmp.cmp( self.result_path1, os.path.join(self.workdir, 'results', 'result_1__foo') ) self.assertTrue(files_are_equal, msg='result files differ') return
def test_get_stored_source_num(self): # we can test whether a source file is stored in a bucket already. bucket = Bucket(self.workdir) self.assertEqual(bucket.get_stored_source_num(self.src_path1), None) self.assertEqual(bucket.get_stored_source_num(self.src_path2), None) shutil.copyfile( self.src_path1, os.path.join(bucket.srcdir, 'source_1')) self.assertEqual(bucket.get_stored_source_num(self.src_path1), 1) self.assertEqual(bucket.get_stored_source_num(self.src_path2), None) shutil.copyfile( self.src_path2, os.path.join(bucket.srcdir, 'source_2')) self.assertEqual(bucket.get_stored_source_num(self.src_path1), 1) self.assertEqual(bucket.get_stored_source_num(self.src_path2), 2) return
def test_init_internal_data(self, tmpdir): # A bucket with same path won't overwrite existing data... bucket1 = Bucket(str(tmpdir)) assert bucket1._get_internal_data() == dict(version=1, curr_src_num=0, curr_repr_num={}) to_set = dict(version=1, curr_src_num=1, curr_repr_num={'1': 2}) bucket1._set_internal_data(to_set) assert bucket1._get_internal_data() == to_set bucket2 = Bucket(str(tmpdir)) assert bucket2._get_internal_data() == to_set
def test_store_representation_no_key(self, cache_env): # we can store sources with their representations bucket = Bucket(str(cache_env.join("cache"))) res = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result1.txt")) source_path = cache_env / "cache" / "sources" / "source_1" result_path = cache_env / "cache" / "repr" / "1" / "1" / "result1.txt" assert res == "1_1" assert source_path.isfile() assert source_path.read() == "source1\n" assert result_path.dirpath().isdir() assert result_path.isfile() assert result_path.read() == "result1\n" assert (cache_env / "cache" / "keys" / "1" / "1.key").isfile() assert (cache_env / "cache" / "keys" / "1" / "1.key").read() == ""
def test_store_several_results(self): bucket = Bucket(self.workdir) bucket.storeResult(self.src_path1, self.result_path1) bucket.storeResult(self.src_path2, self.result_path2) files_are_equal = filecmp.cmp( self.src_path2, os.path.join(self.workdir, 'sources', 'source_2') ) self.assertTrue(files_are_equal, msg='source files differ') files_are_equal = filecmp.cmp( self.result_path2, os.path.join(self.workdir, 'results', 'result_2_default') ) self.assertTrue(files_are_equal, msg='result files differ') return
def test_store_representation_no_key(self, cache_env): # we can store sources with their representations bucket = Bucket(str(cache_env.join("cache"))) res = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result1.txt")) source_path = cache_env / "cache" / "sources" / "source_1" result_path = cache_env / "cache" / "repr" / "1" / "1" / "result1.txt" assert res == "1_1" assert source_path.isfile() assert source_path.read() == "source1\n" assert result_path.dirpath().isdir() assert result_path.isfile() assert result_path.read() == "result1\n" assert (cache_env / "cache" / "keys" / "1" / "1.key").isfile() assert (cache_env / "cache" / "keys" / "1" / "1.key").read() == ""
def test_store_representation_update_result(self, cache_env): # if we send a different representation for the same source # and key, the old representation will be replaced. bucket = Bucket(str(cache_env / "cache")) res1 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key='mykey') res2 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result2.txt"), repr_key='mykey') assert res1 == "1_1" assert res2 == "1_1" result_dir = cache_env / "cache" / "repr" / "1" / "1" assert result_dir.join("result1.txt").exists() is False assert result_dir.join("result2.txt").exists() is True assert result_dir.join("result2.txt").read() == "result2\n"
def test_store_representation_update_result(self, cache_env): # if we send a different representation for the same source # and key, the old representation will be replaced. bucket = Bucket(str(cache_env / "cache")) res1 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key='mykey') res2 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result2.txt"), repr_key='mykey') assert res1 == "1_1" assert res2 == "1_1" result_dir = cache_env / "cache" / "repr" / "1" / "1" assert result_dir.join("result1.txt").exists() is False assert result_dir.join("result2.txt").exists() is True assert result_dir.join("result2.txt").read() == "result2\n"
def test_get_result_path(self): bucket = Bucket(self.workdir) path = bucket.getResultPath(self.src_path1) self.assertEqual(path, None) bucket.storeResult(self.src_path1, self.result_path1) path = bucket.getResultPath(self.src_path1) self.assertNotEqual(path, None) bucket.storeResult(self.src_path1, self.result_path1, suffix='foo') path = bucket.getResultPath(self.src_path1, suffix='foo') self.assertTrue(path.endswith('result_1__foo')) path = bucket.getResultPath(self.src_path1, suffix='bar') self.assertEqual(path, None) return
def test_init_sets_attributes(self, tmpdir): # Main attributes are set properly... bucket = Bucket(str(tmpdir)) assert bucket.srcdir == tmpdir / "sources" assert bucket.resultdir == tmpdir / "repr" assert bucket.keysdir == tmpdir / "keys" assert bucket._data == dict(version=1, curr_src_num=0, curr_repr_num=dict())
def test_curr_repr_num(self, tmpdir): # we can get/set current representation number bucket = Bucket(str(tmpdir)) assert bucket.get_current_repr_num(1) == 0 assert bucket.get_current_repr_num('2') == 0 bucket.set_current_repr_num('1', 12) assert bucket.get_current_repr_num('1') == 12 assert bucket.get_current_repr_num('2') == 0
def test_store_representation_update_result(self): # if we send a different representation for the same source # and key, the old representation will be replaced. bucket = Bucket(self.workdir) res1 = bucket.store_representation( self.src_path1, self.result_path1, repr_key='mykey') res2 = bucket.store_representation( self.src_path1, self.result_path2, repr_key='mykey') exp_stored_repr_dir = os.path.join(self.workdir, b'repr', b'1', b'1') old_stored_repr_data = os.path.join( exp_stored_repr_dir, b'resultfile1') exp_stored_repr_data = os.path.join( exp_stored_repr_dir, b'resultfile2') self.assertEqual(res1, b'1_1') self.assertEqual(res2, b'1_1') self.assertEqual( open(exp_stored_repr_data, 'rb').read(), 'result2\n') self.assertEqual( os.path.exists(old_stored_repr_data), False) return
def test_store_representation_no_key(self): # we can store sources with their representations bucket = Bucket(self.workdir) res = bucket.store_representation(self.src_path1, self.result_path1) exp_stored_src = os.path.join(self.workdir, b'sources', b'source_1') exp_stored_repr_dir = os.path.join(self.workdir, b'repr', b'1', b'1') exp_stored_repr_data = os.path.join( exp_stored_repr_dir, b'resultfile1') exp_stored_key = os.path.join(self.workdir, b'keys', b'1', b'1.key') self.assertTrue(os.path.isfile(exp_stored_src)) self.assertTrue(os.path.isdir(exp_stored_repr_dir)) self.assertTrue(os.path.isfile(exp_stored_repr_data)) self.assertTrue(os.path.isfile(exp_stored_key)) self.assertEqual( open(exp_stored_src, 'rb').read(), b'source1\n') self.assertEqual( open(exp_stored_repr_data, 'rb').read(), b'result1\n') self.assertEqual( open(exp_stored_key, 'rb').read(), b'') self.assertEqual(res, '1_1') return
def test_init_internal_data(self, tmpdir): # A bucket with same path won't overwrite existing data... bucket1 = Bucket(str(tmpdir)) assert bucket1._get_internal_data() == dict( version=1, curr_src_num=0, curr_repr_num={}) to_set = dict(version=1, curr_src_num=1, curr_repr_num={'1': 2}) bucket1._set_internal_data(to_set) assert bucket1._get_internal_data() == to_set bucket2 = Bucket(str(tmpdir)) assert bucket2._get_internal_data() == to_set
def test_init(self): # Make sure, the dir is empty before we mess around... self.assertEqual([], os.listdir(self.workdir)) bucket = Bucket(self.workdir) # Main subdirs were created... for subdirname in ['sources', 'results', 'data']: self.assertTrue( os.path.exists(os.path.join(self.workdir, subdirname)), 'Not found: subdir `%s`' % subdirname ) # Main attributes are set properly... self.assertEqual( bucket.srcdir, os.path.join(self.workdir, 'sources')) self.assertEqual( bucket.resultdir, os.path.join(self.workdir, 'results')) self.assertEqual( bucket._data, dict(version=0, current_num=0)) # A bucket with same path won't overwrite existing data... data = bucket._getInternalData() self.assertEqual(data, dict(version=0, current_num=0)) bucket._setInternalData(dict(version=0, current_num=1)) bucket2 = Bucket(self.workdir) data = bucket2._getInternalData() self.assertEqual(data['current_num'], 1) return
def test_keys(self, cache_env): # we can get a list of all bucket keys in a bucket. bucket = Bucket(str(cache_env)) assert list(bucket.keys()) == [] key1 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key='foo') assert list(bucket.keys()) == [ key1, ] key2 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result2.txt"), repr_key='bar') assert sorted(list(bucket.keys())) == [key1, key2] key3 = bucket.store_representation(str(cache_env / "src1.txt"), str(cache_env / "result3.txt"), repr_key='baz') assert sorted(list(bucket.keys())) == [key1, key2, key3]
def test_keys(self, cache_env): # we can get a list of all bucket keys in a bucket. bucket = Bucket(str(cache_env)) assert list(bucket.keys()) == [] key1 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result1.txt"), repr_key='foo') assert list(bucket.keys()) == [key1, ] key2 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result2.txt"), repr_key='bar') assert sorted(list(bucket.keys())) == [key1, key2] key3 = bucket.store_representation( str(cache_env / "src1.txt"), str(cache_env / "result3.txt"), repr_key='baz') assert sorted(list(bucket.keys())) == [key1, key2, key3]
def test_keys(self): # we can get a list of all bucket keys in a bucket. bucket = Bucket(self.workdir) self.assertEqual(list(bucket.keys()), []) b_key1 = bucket.store_representation( self.src_path1, self.result_path1, repr_key=b'foo') self.assertEqual(list(bucket.keys()), [b_key1]) b_key2 = bucket.store_representation( self.src_path1, self.result_path2, repr_key=b'bar') self.assertEqual( sorted(list(bucket.keys())), sorted([b_key1, b_key2])) b_key3 = bucket.store_representation( self.src_path2, self.result_path1, repr_key=b'baz') self.assertEqual( sorted(list(bucket.keys())), sorted([b_key1, b_key2, b_key3])) return
def test_curr_repr_num(self): # we can get/set current representation number bucket = Bucket(self.workdir) self.assertEqual(bucket.get_current_repr_num(1), 0) self.assertEqual(bucket.get_current_repr_num('2'), 0) bucket.set_current_repr_num('1', 12) self.assertEqual(bucket.get_current_repr_num('1'), 12) self.assertEqual(bucket.get_current_repr_num('2'), 0) return
def test_store_result_twice(self): bucket = Bucket(self.workdir) bucket.storeResult(self.src_path1, self.result_path1, suffix='foo') listing = os.listdir(os.path.join(self.workdir, 'results')) self.assertEqual(listing, ['result_1__foo']) bucket.storeResult(self.src_path1, self.result_path1) bucket.storeResult(self.src_path1, self.result_path2) bucket.storeResult(self.src_path1, self.result_path1) listing = sorted(os.listdir(os.path.join(self.workdir, 'results'))) self.assertEqual(listing, ['result_1__foo', 'result_1_default']) curr_num = bucket.getCurrentNum() self.assertEqual(curr_num, 1) return
def test_get_all_source_paths(self): bucket = Bucket(self.workdir) paths = bucket.getAllSourcePaths() self.assertTrue( isinstance(paths, types.GeneratorType)) paths = list(paths) self.assertEqual(paths, []) bucket.storeResult( self.src_path1, self.result_path1, suffix='foo') paths = list(bucket.getAllSourcePaths()) self.assertTrue(paths[0].endswith('source_1')) self.assertEqual(len(paths), 1) bucket.storeResult( self.src_path2, self.result_path2, suffix='bar') paths = list(bucket.getAllSourcePaths()) self.assertEqual(len(paths), 2) return
def test_get_result_path_from_marker(self): bucket = Bucket(self.workdir) marker1 = bucket.storeResult( self.src_path1, self.result_path1, suffix=None) marker2 = bucket.storeResult( self.src_path1, self.result_path1, suffix='foo') marker3 = bucket.storeResult( self.src_path2, self.result_path1, suffix='foo') path1 = bucket.getResultPathFromMarker(marker1) path2 = bucket.getResultPathFromMarker(marker2, suffix='foo') path3 = bucket.getResultPathFromMarker(marker3, suffix='foo') path4 = bucket.getResultPathFromMarker(marker3, suffix='bar') path5 = bucket.getResultPathFromMarker('1') path6 = bucket.getResultPathFromMarker('3') expected_path1 = os.path.join('results', 'result_1_default') expected_path2 = os.path.join('results', 'result_1__foo') expected_path3 = os.path.join('results', 'result_2__foo') self.assertTrue(path1.endswith(expected_path1)) self.assertTrue(path2.endswith(expected_path2)) self.assertTrue(path3.endswith(expected_path3)) self.assertTrue(path4 is None) self.assertTrue(path5 is not None) self.assertTrue(path6 is None)
def test_get_source_path(self): bucket = Bucket(self.workdir) path, marker = bucket.getSourcePath(self.src_path1) self.assertEqual((path, marker), (None, None)) bucket.storeResult(self.src_path1, self.result_path1) bucket.storeResult(self.src_path2, self.result_path2) path, marker = bucket.getSourcePath(self.src_path1) self.assertEqual( (path, marker), (os.path.join(self.workdir, 'sources', 'source_1'), '1') ) path, marker = bucket.getSourcePath(self.src_path2) self.assertEqual( (path, marker), (os.path.join(self.workdir, 'sources', 'source_2'), '2') ) return
def test_init_creates_subdirs(self, tmpdir): # a new bucket contains certain subdirs and a file Bucket(str(tmpdir)) for filename in ['sources', 'repr', 'keys', 'data']: assert tmpdir.join(filename).exists()
def test_get_representation_unstored(self): # we cannot get unstored representations bucket = Bucket(self.workdir) self.assertEqual(bucket.get_representation(b'1_1'), None) return
def test_curr_num(self): bucket = Bucket(self.workdir) self.assertEqual(bucket.getCurrentNum(), 0) bucket.setCurrentNum(12) self.assertEqual(bucket.getCurrentNum(), 12) return
def test_curr_src_num(self, tmpdir): # we can get/set current source number bucket = Bucket(str(tmpdir)) assert bucket.get_current_source_num() == 0 bucket.set_current_source_num(12) assert bucket.get_current_source_num() == 12
def test_get_representation_unstored(self, tmpdir): # we cannot get unstored representations bucket = Bucket(str(tmpdir.join("cache"))) assert bucket.get_representation("1_1") is None
def test_get_stored_repr_num(self, tmpdir): # we can get a representation number if the repective key is # stored in the bucket already. bucket = Bucket(str(tmpdir.join("cache"))) key_path1 = tmpdir / "cache" / "keys" / "1" / "1.key" key_path2 = tmpdir / "cache" / "keys" / "1" / "2.key" key_path3 = tmpdir / "cache" / "keys" / "2" / "1.key" assert bucket.get_stored_repr_num(1, 'somekey') is None assert bucket.get_stored_repr_num(1, 'otherkey') is None assert bucket.get_stored_repr_num(2, 'somekey') is None assert bucket.get_stored_repr_num(2, 'otherkey') is None key_path1.write('otherkey', ensure=True) assert bucket.get_stored_repr_num(1, 'somekey') is None assert bucket.get_stored_repr_num(1, 'otherkey') == 1 assert bucket.get_stored_repr_num(2, 'somekey') is None assert bucket.get_stored_repr_num(2, 'otherkey') is None key_path2.write('somekey', ensure=True) assert bucket.get_stored_repr_num(1, 'somekey') == 2 assert bucket.get_stored_repr_num(1, 'otherkey') == 1 assert bucket.get_stored_repr_num(2, 'somekey') is None assert bucket.get_stored_repr_num(2, 'otherkey') is None key_path3.write('somekey', ensure=True) assert bucket.get_stored_repr_num(1, 'somekey') == 2 assert bucket.get_stored_repr_num(1, 'otherkey') == 1 assert bucket.get_stored_repr_num(2, 'somekey') == 1 assert bucket.get_stored_repr_num(2, 'otherkey') is None
def test_get_stored_repr_num(self): # we can get a representation number if the repective key is # stored in the bucket already. bucket = Bucket(self.workdir) key_path1 = os.path.join(bucket.keysdir, '1', '1.key') key_path2 = os.path.join(bucket.keysdir, '1', '2.key') key_path3 = os.path.join(bucket.keysdir, '2', '1.key') self.assertEqual(bucket.get_stored_repr_num(1, 'somekey'), None) self.assertEqual(bucket.get_stored_repr_num(1, 'otherkey'), None) self.assertEqual(bucket.get_stored_repr_num(2, 'somekey'), None) self.assertEqual(bucket.get_stored_repr_num(2, 'otherkey'), None) os.makedirs(os.path.dirname(key_path1)) os.makedirs(os.path.dirname(key_path3)) open(key_path1, 'w').write(b'otherkey') self.assertEqual(bucket.get_stored_repr_num(1, 'somekey'), None) self.assertEqual(bucket.get_stored_repr_num(1, 'otherkey'), 1) self.assertEqual(bucket.get_stored_repr_num(2, 'somekey'), None) self.assertEqual(bucket.get_stored_repr_num(2, 'otherkey'), None) open(key_path2, 'w').write(b'somekey') self.assertEqual(bucket.get_stored_repr_num(1, 'somekey'), 2) self.assertEqual(bucket.get_stored_repr_num(1, 'otherkey'), 1) self.assertEqual(bucket.get_stored_repr_num(2, 'somekey'), None) self.assertEqual(bucket.get_stored_repr_num(2, 'otherkey'), None) open(key_path3, 'w').write(b'somekey') self.assertEqual(bucket.get_stored_repr_num(1, 'somekey'), 2) self.assertEqual(bucket.get_stored_repr_num(1, 'otherkey'), 1) self.assertEqual(bucket.get_stored_repr_num(2, 'somekey'), 1) self.assertEqual(bucket.get_stored_repr_num(2, 'otherkey'), None) return