def test_that_a_missing_layout_line_raises_an_error( self, configs_dir, general_section): path = configs_dir / 'no-layout.conf' with path.open('w') as tmp: tmp.write(general_section) tmp.write('[some scenario]\n' 'file1 = 1024\n' 'frags file1 = 2\n') with pytest.raises(ImageConfigError): Image.from_config(pathlib.Path(tmp.name)) path.unlink()
def test_that_file_cannot_be_used_without_number_of_frags( self, configs_dir, general_section): path = configs_dir / 'file-without-frags.conf' with path.open('w') as tmp: tmp.write(general_section) tmp.write('[some scenario]\n' 'file1 = 1024\n' 'frags file2 = 2\n') tmp.write('layout = 1-1, 2.1, 1-2') with pytest.raises(ImageConfigError): Image.from_config(pathlib.Path(tmp.name)) path.unlink()
def test_two_consecutive_image_creations(config_path): image_path = pathlib.Path('/tmp/reproduce-me.dd') config = config_path / 'three-scenarios.conf' image = Image.from_config(config) image.write(image_path) image_digest = hashlib.sha256(image_path.open('rb').read()).hexdigest() image = Image.from_config(config) image.write(pathlib.Path('/tmp/reproduce-me.dd')) assert image_digest == hashlib.sha256(image_path.open('rb').read()).hexdigest()
def test_an_image_with_a_single_scenario(self, test_corpus_path): woodblock.random.seed(13) image = Image(block_size=513) image.add(Scenario('some scenario')) assert image.metadata == { 'block_size': 513, 'seed': 13, 'corpus': str(test_corpus_path), 'scenarios': [{ 'name': 'some scenario', 'files': [] }] }
def test_that_an_invalid_file_fragment_separator_raises_an_error( self, sep, general_section, configs_dir): if sep == '-' or sep == '.': return path = configs_dir / 'invalid-sep.conf' with path.open('w') as tmp: tmp.write(general_section) tmp.write('[some scenario]\nfrags file1 = 2\nfrags_file2 = 2\n') tmp.write('layout = 1-1, 2.1, 1') tmp.write(sep) tmp.write('2\n') with pytest.raises(ImageConfigError): Image.from_config(pathlib.Path(tmp.name)) path.unlink()
def test_an_empty_image(self, test_corpus_path): woodblock.random.seed(13) assert Image(block_size=513).metadata == { 'block_size': 513, 'seed': 13, 'corpus': str(test_corpus_path), 'scenarios': [] }
def test_that_the_intertwine_layout_works(self, config): image = Image.from_config(config['path']) scenario = image.metadata['scenarios'][0] expected = config['expected'] assert len(scenario['files']) == expected['num_files'] for file in scenario['files']: assert expected['min_frags'] <= len( file['fragments']) <= expected['max_frags']
def test_that_the_fillers_have_the_correct_size(self, config): image = Image.from_config(config['config']) block_size = image.metadata['block_size'] scenario = image.metadata['scenarios'][0] expected = config['expected_results'] for file in scenario['files']: if file['original']['type'] == 'filler': assert len(file['fragments']) == 1 assert file['fragments'][0]['size'] % block_size == 0 assert expected['min'] <= file['fragments'][0][ 'size'] / block_size <= expected['max']
def test_an_image_with_a_single_filler_fragment(self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'simple-filler.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) s = Scenario('simple filler fragment') s.add(FillerFragment(1024)) image = Image(block_size=512) image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 1024 assert metadata_path.is_file() meta = json.load(metadata_path.open('r')) assert meta['block_size'] == 512 assert meta['seed'] == 13 assert len(meta['scenarios']) == 1 assert len(meta['scenarios'][0]['files']) == 1 assert meta['scenarios'][0]['files'][0]['original']['type'] == 'filler' assert meta['scenarios'][0]['files'][0]['original']['size'] == 1024 assert len(meta['scenarios'][0]['files'][0]['fragments']) == 1 assert meta['scenarios'][0]['files'][0]['fragments'][0]['number'] == 1 assert meta['scenarios'][0]['files'][0]['fragments'][0]['size'] == 1024 assert meta['scenarios'][0]['files'][0]['fragments'][0][ 'file_offsets'] == { 'start': 0, 'end': 1024 } assert meta['scenarios'][0]['files'][0]['fragments'][0][ 'image_offsets'] == { 'start': 0, 'end': 1024 } os.remove(image_path) os.remove(metadata_path)
def test_an_image_padding_with_a_single_zeroes_fragment( self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'simple-zeroes-with-padding.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) s = Scenario('simple zeroes with padding') s.add(ZeroesFragment(1000)) image = Image(block_size=512, padding_generator=lambda x: b'A' * x) image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 1024 assert image_path.open('rb').read() == b''.join( (b'\x00' * 1000, b'A' * 24)) assert metadata_path.is_file() metadata = _replace_uuids(json.load(metadata_path.open('r'))) assert metadata == { 'block_size': 512, 'corpus': str(test_corpus_path), 'seed': woodblock.random.get_seed(), 'scenarios': [{ 'name': 'simple zeroes with padding', 'files': [{ 'original': { 'type': 'filler', 'path': 'zeroes', 'size': 1000, 'id': 'uuid', 'sha256': '541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53' }, 'fragments': [{ 'number': 1, 'size': 1000, 'sha256': '541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53', 'file_offsets': { 'start': 0, 'end': 1000 }, 'image_offsets': { 'start': 0, 'end': 1000 } }] }] }] } os.remove(image_path) os.remove(metadata_path)
def test_that_an_empty_with_an_empty_scenario_can_be_written(self): s = Scenario('empty') image = Image() image.add(s) f = io.BytesIO() image.write(f) f.seek(0) assert len(f.read()) == 0
def test_that_a_scenario_can_be_written(self): s = Scenario('scenario') s.add(ZeroesFragment(512)) image = Image() image.add(s) f = io.BytesIO() image.write(f) f.seek(0) assert len(f.read()) == 512
def test_an_image_with_a_single_file_fragment(self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'simple-file.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) s = Scenario('simple file fragment') s.add(File(test_corpus_path / '512').as_fragment()) image = Image(block_size=512) image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 512 assert image_path.open('rb').read() == b'A' * 512 assert metadata_path.is_file() assert _replace_uuids(json.load(metadata_path.open('r'))) == { 'block_size': 512, 'corpus': str(test_corpus_path), 'seed': woodblock.random.get_seed(), 'scenarios': [{ 'name': 'simple file fragment', 'files': [{ 'original': { 'type': 'file', 'path': str((HERE.parent / 'data' / 'corpus' / '512').relative_to(get_corpus())), 'size': 512, 'id': 'uuid', 'sha256': '32beecb58a128af8248504600bd203dcc676adf41045300485655e6b8780a01d' }, 'fragments': [{ 'number': 1, 'size': 512, 'sha256': '32beecb58a128af8248504600bd203dcc676adf41045300485655e6b8780a01d', 'file_offsets': { 'start': 0, 'end': 512 }, 'image_offsets': { 'start': 0, 'end': 512 } }] }] }] } os.remove(image_path) os.remove(metadata_path)
def test_an_image_with_a_single_zeroes_fragment(self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'simple-zeroes.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) s = Scenario('simple zeroes') s.add(ZeroesFragment(1024)) image = Image() image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 1024 assert image_path.open('rb').read() == b'\x00' * 1024 assert metadata_path.is_file() assert _replace_uuids(json.load(metadata_path.open('r'))) == { 'block_size': 512, 'corpus': str(test_corpus_path), 'seed': woodblock.random.get_seed(), 'scenarios': [{ 'name': 'simple zeroes', 'files': [{ 'original': { 'type': 'filler', 'path': 'zeroes', 'size': 1024, 'id': 'uuid', 'sha256': '5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef' }, 'fragments': [{ 'number': 1, 'size': 1024, 'sha256': '5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef', 'file_offsets': { 'start': 0, 'end': 1024 }, 'image_offsets': { 'start': 0, 'end': 1024 } }] }] }] } os.remove(image_path) os.remove(metadata_path)
def test_that_correct_padding_is_applied(self, fragment_size, expected_padding_size): s = Scenario('scenario') s.add(ZeroesFragment(fragment_size)) padder = FakePaddingGenerator() image = Image(padding_generator=padder.generate) image.add(s) f = io.BytesIO() image.write(f) f.seek(0) data = f.read() assert len(data) == ceil(fragment_size / 512) * 512 assert padder.size == expected_padding_size if expected_padding_size > 0: assert data[ -expected_padding_size:] == b'A' * expected_padding_size
def test_an_image_with_multiple_scenarios(self, test_corpus_path): woodblock.random.seed(13) image = Image(block_size=513) image.add(Scenario('first scenario')) image.add(Scenario('second scenario')) image.add(Scenario('third scenario')) assert image.metadata == { 'block_size': 513, 'seed': 13, 'corpus': str(test_corpus_path), 'scenarios': [{ 'name': 'first scenario', 'files': [] }, { 'name': 'second scenario', 'files': [] }, { 'name': 'third scenario', 'files': [] }] }
def test_that_the_correct_number_of_files_per_scenario_is_present( self, config, num_expected_files): image = Image.from_config(config) files_per_scenario = tuple( len(s['files']) for s in image.metadata['scenarios']) assert files_per_scenario == num_expected_files
def test_that_the_correct_number_of_scenarios_is_present( self, config, num_expected_scenarios): image = Image.from_config(config) assert len(image.metadata['scenarios']) == num_expected_scenarios
def test_that_the_seed_is_set(self, minimal_config): Image.from_config(minimal_config) assert woodblock.random.get_seed() == 123
def test_that_a_missing_corpus_option_raises_an_error(self, path): with pytest.raises(ImageConfigError): Image.from_config(path)
def test_that_invalid_intertwine_scenarios_raise_an_error(self, config): with pytest.raises(ImageConfigError): Image.from_config(config)
def test_that_an_empty_image_can_be_written(self): f = io.BytesIO() image = Image() image.write(f) f.seek(0) assert len(f.read()) == 0
def test_that_an_invalid_path_raises_an_error(self): with pytest.raises(FileNotFoundError): Image.from_config(pathlib.Path('/some/path'))
def test_that_invalid_min_max_blocks_raise_an_error(self, config): with pytest.raises(ImageConfigError): Image.from_config(config)
def test_an_image_with_two_reversed_file_fragments_and_three_filler_fragments(self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'simple-fragmentation.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) file_path = test_corpus_path / '4096' frags = File(file_path).fragment_evenly(num_fragments=2) s = Scenario('simple fragmentation with fillers') s.add(RandomDataFragment(512)) s.add(frags[1]) s.add(ZeroesFragment(512)) s.add(frags[0]) s.add(RandomDataFragment(512)) image = Image(block_size=512) image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 5632 assert metadata_path.is_file() meta = _replace_uuids(json.load(metadata_path.open('r'))) assert meta['block_size'] == 512 assert meta['corpus'] == str(test_corpus_path) assert meta['seed'] == 13 assert len(meta['scenarios']) == 1 sc_meta = meta['scenarios'][0] assert sc_meta['name'] == 'simple fragmentation with fillers' assert len(sc_meta['files']) == 4 assert sc_meta['files'][0]['original']['type'] == 'filler' assert sc_meta['files'][0]['original']['path'] == 'random' assert sc_meta['files'][0]['original']['size'] == 512 assert len(sc_meta['files'][0]['fragments']) == 1 assert sc_meta['files'][0]['fragments'][0]['number'] == 1 assert sc_meta['files'][0]['fragments'][0]['size'] == 512 assert sc_meta['files'][0]['fragments'][0]['file_offsets'] == {'start': 0, 'end': 512} assert sc_meta['files'][0]['fragments'][0]['image_offsets'] == {'start': 0, 'end': 512} assert sc_meta['files'][1] == { 'original': {'type': 'file', 'path': str(file_path.relative_to(get_corpus())), 'size': 4096, 'id': 'uuid', 'sha256': '0ec91e13ced59cfc1f297cfedd8c595a6200ac2b8c99bcc321e8c68cf1f166a0'}, 'fragments': [{'number': 1, 'size': 2048, 'sha256': '291b808471ca4772e260dd50604e93082c22fbcf821fa3db7531e51343473717', 'file_offsets': {'start': 0, 'end': 2048}, 'image_offsets': {'start': 3072, 'end': 5120}}, {'number': 2, 'size': 2048, 'sha256': '570dc7f7ceea7fa7204588456fd00e53b0abf1cca9d35cc074383e8dcc418114', 'file_offsets': {'start': 2048, 'end': 4096}, 'image_offsets': {'start': 512, 'end': 2560}} ]} assert sc_meta['files'][2] == { 'original': {'type': 'filler', 'path': 'zeroes', 'size': 512, 'id': 'uuid', 'sha256': '076a27c79e5ace2a3d47f9dd2e83e4ff6ea8872b3c2218f66c92b89b55f36560'}, 'fragments': [{'number': 1, 'size': 512, 'sha256': '076a27c79e5ace2a3d47f9dd2e83e4ff6ea8872b3c2218f66c92b89b55f36560', 'file_offsets': {'start': 0, 'end': 512}, 'image_offsets': {'start': 2560, 'end': 3072}}]} assert sc_meta['files'][3]['original']['type'] == 'filler' assert sc_meta['files'][3]['original']['path'] == 'random' assert sc_meta['files'][3]['original']['size'] == 512 assert len(sc_meta['files'][3]['fragments']) == 1 assert sc_meta['files'][3]['fragments'][0]['number'] == 1 assert sc_meta['files'][3]['fragments'][0]['size'] == 512 assert sc_meta['files'][3]['fragments'][0]['file_offsets'] == {'start': 0, 'end': 512} assert sc_meta['files'][3]['fragments'][0]['image_offsets'] == {'start': 5120, 'end': 5632} os.remove(image_path) os.remove(metadata_path)
def test_an_image_padding_and_two_zeroes_fragments(self, test_corpus_path): woodblock.random.seed(13) woodblock.file.corpus(test_corpus_path) image_path = HERE / 'two-zeroes-with-padding.dd' metadata_path = pathlib.Path('.'.join((str(image_path), 'json'))) s = Scenario('two zeroes with padding') s.add(ZeroesFragment(1000)) s.add(ZeroesFragment(2048)) image = Image(block_size=512, padding_generator=lambda x: b'A' * x) image.add(s) image.write(image_path) assert image_path.is_file() assert image_path.stat().st_size == 1024 + 2048 assert image_path.open('rb').read() == b''.join( (b'\x00' * 1000, b'A' * 24, b'\x00' * 2048)) assert metadata_path.is_file() metadata = _replace_uuids(json.load(open(metadata_path, 'r'))) assert metadata == { 'block_size': 512, 'corpus': str(test_corpus_path), 'seed': woodblock.random.get_seed(), 'scenarios': [{ 'name': 'two zeroes with padding', 'files': [{ 'original': { 'type': 'filler', 'path': 'zeroes', 'size': 1000, 'id': 'uuid', 'sha256': '541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53' }, 'fragments': [{ 'number': 1, 'size': 1000, 'sha256': '541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53', 'file_offsets': { 'start': 0, 'end': 1000 }, 'image_offsets': { 'start': 0, 'end': 1000 } }] }, { 'original': { 'type': 'filler', 'path': 'zeroes', 'size': 2048, 'id': 'uuid', 'sha256': 'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad' }, 'fragments': [{ 'number': 1, 'size': 2048, 'sha256': 'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad', 'file_offsets': { 'start': 0, 'end': 2048 }, 'image_offsets': { 'start': 1024, 'end': 3072 } }] }] }] } os.remove(image_path) os.remove(metadata_path)
def test_that_fillers_can_be_added(self, config): image = Image.from_config(config) scenario = image.metadata['scenarios'][0] assert len(scenario['files']) == 3 assert sum(1 for f in scenario['files'] if f['original']['type'] == 'filler') == 2
def test_config(self, config_path): config = config_path / 'three-scenarios.conf' image = Image.from_config(config) image.write(pathlib.Path('/tmp/a.dd')) meta = image.metadata fragment_order = _get_fragment_order_from_metadata(image.metadata) file_ids = dict() assert meta['block_size'] == 512 assert meta['seed'] == 123 assert len(meta['scenarios']) == 3 s1 = meta['scenarios'][0] assert s1['name'] == 'first scenario' assert len(s1['files']) == 3 file_ids['s1'] = { 1: fragment_order[0][0], 2: fragment_order[3][0], 3: fragment_order[1][0] } s2 = meta['scenarios'][1] assert s2['name'] == 'second scenario' assert len(s2['files']) == 2 file_ids['s2'] = {1: fragment_order[5][0], 2: fragment_order[6][0]} file_1 = _get_file_with_id(s2, file_ids['s2'][1]) file_2 = _get_file_with_id(s2, file_ids['s2'][2]) assert file_1['original']['path'] == '2000' assert file_2['original']['path'] == '1024' s3 = meta['scenarios'][2] assert s3['name'] == 'third scenario' assert len(s3['files']) == 3 file_ids['s3'] = { 2: fragment_order[8][0], 3: fragment_order[9][0], 4: fragment_order[11][0] } file_1 = _get_file_with_id(s3, file_ids['s3'][2]) file_3 = _get_file_with_id(s3, file_ids['s3'][3]) file_4 = _get_file_with_id(s3, file_ids['s3'][4]) assert file_1['original']['path'] == '4096' assert file_3['original']['path'] == 'letters/ascii_letters' assert file_4['original']['path'].startswith('letters/') expected_fragment_order = ((file_ids['s1'][1], 1), (file_ids['s1'][3], 1), (file_ids['s1'][1], 2), (file_ids['s1'][2], 1), (file_ids['s1'][1], 3), (file_ids['s2'][1], 1), (file_ids['s2'][2], 1), (file_ids['s2'][1], 3), (file_ids['s3'][2], 1), (file_ids['s3'][3], 1), (file_ids['s3'][2], 2), (file_ids['s3'][4], 2), (file_ids['s3'][3], 2), (file_ids['s3'][4], 1), (file_ids['s3'][2], 4), (file_ids['s3'][2], 3)) assert fragment_order == expected_fragment_order