def test_that_a_missing_layout_line_raises_an_error(
self, configs_dir, general_section):
path = configs_dir / 'no-layout.conf'
with path.open('w') as tmp:
tmp.write(general_section)
tmp.write('[some scenario]\n' 'file1 = 1024\n' 'frags file1 = 2\n')
with pytest.raises(ImageConfigError):
Image.from_config(pathlib.Path(tmp.name))
path.unlink()
def test_that_file_cannot_be_used_without_number_of_frags(
self, configs_dir, general_section):
path = configs_dir / 'file-without-frags.conf'
with path.open('w') as tmp:
tmp.write(general_section)
tmp.write('[some scenario]\n' 'file1 = 1024\n' 'frags file2 = 2\n')
tmp.write('layout = 1-1, 2.1, 1-2')
with pytest.raises(ImageConfigError):
Image.from_config(pathlib.Path(tmp.name))
path.unlink()
def test_two_consecutive_image_creations(config_path):
image_path = pathlib.Path('/tmp/reproduce-me.dd')
config = config_path / 'three-scenarios.conf'
image = Image.from_config(config)
image.write(image_path)
image_digest = hashlib.sha256(image_path.open('rb').read()).hexdigest()
image = Image.from_config(config)
image.write(pathlib.Path('/tmp/reproduce-me.dd'))
assert image_digest == hashlib.sha256(image_path.open('rb').read()).hexdigest()
def test_an_image_with_a_single_scenario(self, test_corpus_path):
woodblock.random.seed(13)
image = Image(block_size=513)
image.add(Scenario('some scenario'))
assert image.metadata == {
'block_size': 513,
'seed': 13,
'corpus': str(test_corpus_path),
'scenarios': [{
'name': 'some scenario',
'files': []
}]
}
def test_that_an_invalid_file_fragment_separator_raises_an_error(
self, sep, general_section, configs_dir):
if sep == '-' or sep == '.':
return
path = configs_dir / 'invalid-sep.conf'
with path.open('w') as tmp:
tmp.write(general_section)
tmp.write('[some scenario]\nfrags file1 = 2\nfrags_file2 = 2\n')
tmp.write('layout = 1-1, 2.1, 1')
tmp.write(sep)
tmp.write('2\n')
with pytest.raises(ImageConfigError):
Image.from_config(pathlib.Path(tmp.name))
path.unlink()
def test_an_empty_image(self, test_corpus_path):
woodblock.random.seed(13)
assert Image(block_size=513).metadata == {
'block_size': 513,
'seed': 13,
'corpus': str(test_corpus_path),
'scenarios': []
}
def test_that_the_intertwine_layout_works(self, config):
image = Image.from_config(config['path'])
scenario = image.metadata['scenarios'][0]
expected = config['expected']
assert len(scenario['files']) == expected['num_files']
for file in scenario['files']:
assert expected['min_frags'] <= len(
file['fragments']) <= expected['max_frags']
def test_that_the_fillers_have_the_correct_size(self, config):
image = Image.from_config(config['config'])
block_size = image.metadata['block_size']
scenario = image.metadata['scenarios'][0]
expected = config['expected_results']
for file in scenario['files']:
if file['original']['type'] == 'filler':
assert len(file['fragments']) == 1
assert file['fragments'][0]['size'] % block_size == 0
assert expected['min'] <= file['fragments'][0][
'size'] / block_size <= expected['max']
def test_an_image_with_a_single_filler_fragment(self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'simple-filler.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
s = Scenario('simple filler fragment')
s.add(FillerFragment(1024))
image = Image(block_size=512)
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 1024
assert metadata_path.is_file()
meta = json.load(metadata_path.open('r'))
assert meta['block_size'] == 512
assert meta['seed'] == 13
assert len(meta['scenarios']) == 1
assert len(meta['scenarios'][0]['files']) == 1
assert meta['scenarios'][0]['files'][0]['original']['type'] == 'filler'
assert meta['scenarios'][0]['files'][0]['original']['size'] == 1024
assert len(meta['scenarios'][0]['files'][0]['fragments']) == 1
assert meta['scenarios'][0]['files'][0]['fragments'][0]['number'] == 1
assert meta['scenarios'][0]['files'][0]['fragments'][0]['size'] == 1024
assert meta['scenarios'][0]['files'][0]['fragments'][0][
'file_offsets'] == {
'start': 0,
'end': 1024
}
assert meta['scenarios'][0]['files'][0]['fragments'][0][
'image_offsets'] == {
'start': 0,
'end': 1024
}
os.remove(image_path)
os.remove(metadata_path)
def test_an_image_padding_with_a_single_zeroes_fragment(
self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'simple-zeroes-with-padding.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
s = Scenario('simple zeroes with padding')
s.add(ZeroesFragment(1000))
image = Image(block_size=512, padding_generator=lambda x: b'A' * x)
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 1024
assert image_path.open('rb').read() == b''.join(
(b'\x00' * 1000, b'A' * 24))
assert metadata_path.is_file()
metadata = _replace_uuids(json.load(metadata_path.open('r')))
assert metadata == {
'block_size':
512,
'corpus':
str(test_corpus_path),
'seed':
woodblock.random.get_seed(),
'scenarios': [{
'name':
'simple zeroes with padding',
'files': [{
'original': {
'type':
'filler',
'path':
'zeroes',
'size':
1000,
'id':
'uuid',
'sha256':
'541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53'
},
'fragments': [{
'number': 1,
'size': 1000,
'sha256':
'541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53',
'file_offsets': {
'start': 0,
'end': 1000
},
'image_offsets': {
'start': 0,
'end': 1000
}
}]
}]
}]
}
os.remove(image_path)
os.remove(metadata_path)
def test_that_an_empty_with_an_empty_scenario_can_be_written(self):
s = Scenario('empty')
image = Image()
image.add(s)
f = io.BytesIO()
image.write(f)
f.seek(0)
assert len(f.read()) == 0
def test_that_a_scenario_can_be_written(self):
s = Scenario('scenario')
s.add(ZeroesFragment(512))
image = Image()
image.add(s)
f = io.BytesIO()
image.write(f)
f.seek(0)
assert len(f.read()) == 512
def test_an_image_with_a_single_file_fragment(self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'simple-file.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
s = Scenario('simple file fragment')
s.add(File(test_corpus_path / '512').as_fragment())
image = Image(block_size=512)
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 512
assert image_path.open('rb').read() == b'A' * 512
assert metadata_path.is_file()
assert _replace_uuids(json.load(metadata_path.open('r'))) == {
'block_size':
512,
'corpus':
str(test_corpus_path),
'seed':
woodblock.random.get_seed(),
'scenarios': [{
'name':
'simple file fragment',
'files': [{
'original': {
'type':
'file',
'path':
str((HERE.parent / 'data' / 'corpus' /
'512').relative_to(get_corpus())),
'size':
512,
'id':
'uuid',
'sha256':
'32beecb58a128af8248504600bd203dcc676adf41045300485655e6b8780a01d'
},
'fragments': [{
'number': 1,
'size': 512,
'sha256':
'32beecb58a128af8248504600bd203dcc676adf41045300485655e6b8780a01d',
'file_offsets': {
'start': 0,
'end': 512
},
'image_offsets': {
'start': 0,
'end': 512
}
}]
}]
}]
}
os.remove(image_path)
os.remove(metadata_path)
def test_an_image_with_a_single_zeroes_fragment(self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'simple-zeroes.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
s = Scenario('simple zeroes')
s.add(ZeroesFragment(1024))
image = Image()
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 1024
assert image_path.open('rb').read() == b'\x00' * 1024
assert metadata_path.is_file()
assert _replace_uuids(json.load(metadata_path.open('r'))) == {
'block_size':
512,
'corpus':
str(test_corpus_path),
'seed':
woodblock.random.get_seed(),
'scenarios': [{
'name':
'simple zeroes',
'files': [{
'original': {
'type':
'filler',
'path':
'zeroes',
'size':
1024,
'id':
'uuid',
'sha256':
'5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef'
},
'fragments': [{
'number': 1,
'size': 1024,
'sha256':
'5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef',
'file_offsets': {
'start': 0,
'end': 1024
},
'image_offsets': {
'start': 0,
'end': 1024
}
}]
}]
}]
}
os.remove(image_path)
os.remove(metadata_path)
def test_that_correct_padding_is_applied(self, fragment_size,
expected_padding_size):
s = Scenario('scenario')
s.add(ZeroesFragment(fragment_size))
padder = FakePaddingGenerator()
image = Image(padding_generator=padder.generate)
image.add(s)
f = io.BytesIO()
image.write(f)
f.seek(0)
data = f.read()
assert len(data) == ceil(fragment_size / 512) * 512
assert padder.size == expected_padding_size
if expected_padding_size > 0:
assert data[
-expected_padding_size:] == b'A' * expected_padding_size
def test_an_image_with_multiple_scenarios(self, test_corpus_path):
woodblock.random.seed(13)
image = Image(block_size=513)
image.add(Scenario('first scenario'))
image.add(Scenario('second scenario'))
image.add(Scenario('third scenario'))
assert image.metadata == {
'block_size':
513,
'seed':
13,
'corpus':
str(test_corpus_path),
'scenarios': [{
'name': 'first scenario',
'files': []
}, {
'name': 'second scenario',
'files': []
}, {
'name': 'third scenario',
'files': []
}]
}
def test_that_the_correct_number_of_files_per_scenario_is_present(
self, config, num_expected_files):
image = Image.from_config(config)
files_per_scenario = tuple(
len(s['files']) for s in image.metadata['scenarios'])
assert files_per_scenario == num_expected_files
def test_that_the_correct_number_of_scenarios_is_present(
self, config, num_expected_scenarios):
image = Image.from_config(config)
assert len(image.metadata['scenarios']) == num_expected_scenarios
def test_that_the_seed_is_set(self, minimal_config):
Image.from_config(minimal_config)
assert woodblock.random.get_seed() == 123
def test_that_a_missing_corpus_option_raises_an_error(self, path):
with pytest.raises(ImageConfigError):
Image.from_config(path)
def test_that_invalid_intertwine_scenarios_raise_an_error(self, config):
with pytest.raises(ImageConfigError):
Image.from_config(config)
def test_that_an_empty_image_can_be_written(self):
f = io.BytesIO()
image = Image()
image.write(f)
f.seek(0)
assert len(f.read()) == 0
def test_that_an_invalid_path_raises_an_error(self):
with pytest.raises(FileNotFoundError):
Image.from_config(pathlib.Path('/some/path'))
def test_that_invalid_min_max_blocks_raise_an_error(self, config):
with pytest.raises(ImageConfigError):
Image.from_config(config)
def test_an_image_with_two_reversed_file_fragments_and_three_filler_fragments(self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'simple-fragmentation.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
file_path = test_corpus_path / '4096'
frags = File(file_path).fragment_evenly(num_fragments=2)
s = Scenario('simple fragmentation with fillers')
s.add(RandomDataFragment(512))
s.add(frags[1])
s.add(ZeroesFragment(512))
s.add(frags[0])
s.add(RandomDataFragment(512))
image = Image(block_size=512)
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 5632
assert metadata_path.is_file()
meta = _replace_uuids(json.load(metadata_path.open('r')))
assert meta['block_size'] == 512
assert meta['corpus'] == str(test_corpus_path)
assert meta['seed'] == 13
assert len(meta['scenarios']) == 1
sc_meta = meta['scenarios'][0]
assert sc_meta['name'] == 'simple fragmentation with fillers'
assert len(sc_meta['files']) == 4
assert sc_meta['files'][0]['original']['type'] == 'filler'
assert sc_meta['files'][0]['original']['path'] == 'random'
assert sc_meta['files'][0]['original']['size'] == 512
assert len(sc_meta['files'][0]['fragments']) == 1
assert sc_meta['files'][0]['fragments'][0]['number'] == 1
assert sc_meta['files'][0]['fragments'][0]['size'] == 512
assert sc_meta['files'][0]['fragments'][0]['file_offsets'] == {'start': 0, 'end': 512}
assert sc_meta['files'][0]['fragments'][0]['image_offsets'] == {'start': 0, 'end': 512}
assert sc_meta['files'][1] == {
'original': {'type': 'file', 'path': str(file_path.relative_to(get_corpus())), 'size': 4096, 'id': 'uuid',
'sha256': '0ec91e13ced59cfc1f297cfedd8c595a6200ac2b8c99bcc321e8c68cf1f166a0'},
'fragments': [{'number': 1, 'size': 2048,
'sha256': '291b808471ca4772e260dd50604e93082c22fbcf821fa3db7531e51343473717',
'file_offsets': {'start': 0, 'end': 2048}, 'image_offsets': {'start': 3072, 'end': 5120}},
{'number': 2, 'size': 2048,
'sha256': '570dc7f7ceea7fa7204588456fd00e53b0abf1cca9d35cc074383e8dcc418114',
'file_offsets': {'start': 2048, 'end': 4096}, 'image_offsets': {'start': 512, 'end': 2560}}
]}
assert sc_meta['files'][2] == {
'original': {'type': 'filler', 'path': 'zeroes', 'size': 512, 'id': 'uuid',
'sha256': '076a27c79e5ace2a3d47f9dd2e83e4ff6ea8872b3c2218f66c92b89b55f36560'},
'fragments': [{'number': 1, 'size': 512,
'sha256': '076a27c79e5ace2a3d47f9dd2e83e4ff6ea8872b3c2218f66c92b89b55f36560',
'file_offsets': {'start': 0, 'end': 512}, 'image_offsets': {'start': 2560, 'end': 3072}}]}
assert sc_meta['files'][3]['original']['type'] == 'filler'
assert sc_meta['files'][3]['original']['path'] == 'random'
assert sc_meta['files'][3]['original']['size'] == 512
assert len(sc_meta['files'][3]['fragments']) == 1
assert sc_meta['files'][3]['fragments'][0]['number'] == 1
assert sc_meta['files'][3]['fragments'][0]['size'] == 512
assert sc_meta['files'][3]['fragments'][0]['file_offsets'] == {'start': 0, 'end': 512}
assert sc_meta['files'][3]['fragments'][0]['image_offsets'] == {'start': 5120, 'end': 5632}
os.remove(image_path)
os.remove(metadata_path)
def test_an_image_padding_and_two_zeroes_fragments(self, test_corpus_path):
woodblock.random.seed(13)
woodblock.file.corpus(test_corpus_path)
image_path = HERE / 'two-zeroes-with-padding.dd'
metadata_path = pathlib.Path('.'.join((str(image_path), 'json')))
s = Scenario('two zeroes with padding')
s.add(ZeroesFragment(1000))
s.add(ZeroesFragment(2048))
image = Image(block_size=512, padding_generator=lambda x: b'A' * x)
image.add(s)
image.write(image_path)
assert image_path.is_file()
assert image_path.stat().st_size == 1024 + 2048
assert image_path.open('rb').read() == b''.join(
(b'\x00' * 1000, b'A' * 24, b'\x00' * 2048))
assert metadata_path.is_file()
metadata = _replace_uuids(json.load(open(metadata_path, 'r')))
assert metadata == {
'block_size':
512,
'corpus':
str(test_corpus_path),
'seed':
woodblock.random.get_seed(),
'scenarios': [{
'name':
'two zeroes with padding',
'files': [{
'original': {
'type':
'filler',
'path':
'zeroes',
'size':
1000,
'id':
'uuid',
'sha256':
'541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53'
},
'fragments': [{
'number': 1,
'size': 1000,
'sha256':
'541b3e9daa09b20bf85fa273e5cbd3e80185aa4ec298e765db87742b70138a53',
'file_offsets': {
'start': 0,
'end': 1000
},
'image_offsets': {
'start': 0,
'end': 1000
}
}]
}, {
'original': {
'type':
'filler',
'path':
'zeroes',
'size':
2048,
'id':
'uuid',
'sha256':
'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad'
},
'fragments': [{
'number': 1,
'size': 2048,
'sha256':
'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad',
'file_offsets': {
'start': 0,
'end': 2048
},
'image_offsets': {
'start': 1024,
'end': 3072
}
}]
}]
}]
}
os.remove(image_path)
os.remove(metadata_path)
def test_that_fillers_can_be_added(self, config):
image = Image.from_config(config)
scenario = image.metadata['scenarios'][0]
assert len(scenario['files']) == 3
assert sum(1 for f in scenario['files']
if f['original']['type'] == 'filler') == 2
def test_config(self, config_path):
config = config_path / 'three-scenarios.conf'
image = Image.from_config(config)
image.write(pathlib.Path('/tmp/a.dd'))
meta = image.metadata
fragment_order = _get_fragment_order_from_metadata(image.metadata)
file_ids = dict()
assert meta['block_size'] == 512
assert meta['seed'] == 123
assert len(meta['scenarios']) == 3
s1 = meta['scenarios'][0]
assert s1['name'] == 'first scenario'
assert len(s1['files']) == 3
file_ids['s1'] = {
1: fragment_order[0][0],
2: fragment_order[3][0],
3: fragment_order[1][0]
}
s2 = meta['scenarios'][1]
assert s2['name'] == 'second scenario'
assert len(s2['files']) == 2
file_ids['s2'] = {1: fragment_order[5][0], 2: fragment_order[6][0]}
file_1 = _get_file_with_id(s2, file_ids['s2'][1])
file_2 = _get_file_with_id(s2, file_ids['s2'][2])
assert file_1['original']['path'] == '2000'
assert file_2['original']['path'] == '1024'
s3 = meta['scenarios'][2]
assert s3['name'] == 'third scenario'
assert len(s3['files']) == 3
file_ids['s3'] = {
2: fragment_order[8][0],
3: fragment_order[9][0],
4: fragment_order[11][0]
}
file_1 = _get_file_with_id(s3, file_ids['s3'][2])
file_3 = _get_file_with_id(s3, file_ids['s3'][3])
file_4 = _get_file_with_id(s3, file_ids['s3'][4])
assert file_1['original']['path'] == '4096'
assert file_3['original']['path'] == 'letters/ascii_letters'
assert file_4['original']['path'].startswith('letters/')
expected_fragment_order = ((file_ids['s1'][1],
1), (file_ids['s1'][3],
1), (file_ids['s1'][1],
2), (file_ids['s1'][2],
1), (file_ids['s1'][1], 3),
(file_ids['s2'][1],
1), (file_ids['s2'][2],
1), (file_ids['s2'][1],
3), (file_ids['s3'][2],
1), (file_ids['s3'][3], 1),
(file_ids['s3'][2],
2), (file_ids['s3'][4],
2), (file_ids['s3'][3],
2), (file_ids['s3'][4], 1),
(file_ids['s3'][2], 4), (file_ids['s3'][2],
3))
assert fragment_order == expected_fragment_order
