def test_file_with_signature_match_is_carved(scan_environment):
fn = pathlib.Path("unpackers") / "combined" / "double-gimpbrush.bla"
fn_abs = testdata_dir / fn
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
assert len(scan_environment.resultqueue.queue) == 3
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
result3 = scan_environment.resultqueue.get()
# unpack file at root has absolute path
assert result1.filename == fn_abs
assert result2.filename.name == 'unpacked.gimpbrush'
assert result2.filename.parent.parent == pathlib.Path('.')
assert result3.filename.name == 'unpacked.gimpbrush'
assert result3.filename.parent.parent == pathlib.Path('.')
def test_file_without_features_is_carved(scan_environment):
fn = pathlib.Path("unpackers") / "combined" / "kernelconfig-gif.bla"
fn_abs = testdata_dir / fn
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
print(scan_environment.resultqueue.queue)
assert len(scan_environment.resultqueue.queue) == 3
# assertlen(scan_environment.resultqueue.queue) == 4
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
result3 = scan_environment.resultqueue.get()
# result4 = scan_environment.resultqueue.get()
# first result is for the file we queued and has an absolute path
assert result1.filename == fn_abs
# second result is the one matched by signature
assert result2.filename.name == 'unpacked.gif'
assert result2.filename.parent.parent == pathlib.Path('.')
# third result is synthesized
# gif_offset = 202554
gif_offset = result1.unpackedfiles[0]['offset']
assert result3.filename.name == \
'unpacked-0x%x-0x%x' % (0,gif_offset-1)
assert 'kernel configuration' in result3.labels
def test_featureless_file_is_unpacked(scan_environment):
fn = pathlib.Path("unpackers") / "ihex" / "example.txt"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scanjob.set_scanenvironment(scan_environment)
scanjob.initialize()
unpacker = UnpackManager(scan_environment.unpackdirectory)
scanjob.prepare_for_unpacking()
scanjob.check_for_valid_extension(unpacker)
assert fileresult.labels == set()
scanjob.check_for_signatures(unpacker)
assert fileresult.labels == set()
assert fileresult.unpackedfiles == []
scanjob.carve_file_data(unpacker)
assert fileresult.unpackedfiles == []
fileresult.labels.add('text')
scanjob.check_entire_file(unpacker)
assert len(fileresult.unpackedfiles) == 1
j = scan_environment.scanfilequeue.get()
expected_extracted_fn = pathlib.Path('.') / \
("%s-0x%08x-ihex-1" % (fn.name, 0)) / "unpacked-from-ihex"
assert j.fileresult.filename == expected_extracted_fn
assertUnpackedPathExists(scan_environment, j.fileresult.filename)
def create_fileresult_for_path(unpackdir, path, labels, calculate_size=False):
parentlabels = set()
parent = FileResult(None, path.parent, parentlabels)
fr = FileResult(parent, path, labels)
if calculate_size:
fp = pathlib.Path(unpackdir) / path
fr.set_filesize(fp.stat().st_size)
return fr
def fileresult(basedir, rel_path, labels, calculate_size=True):
parentlabels = set()
parent = FileResult(None, rel_path.parent, parentlabels)
fr = FileResult(parent, rel_path, labels)
if calculate_size:
fp = pathlib.Path(basedir) / rel_path
fr.set_filesize(fp.stat().st_size)
return fr
def create_fileresult_for_path(unpackdir,
path,
labels=set(),
calculate_size=True):
parentlabels = set()
fp = pathlib.Path(unpackdir) / path
fr = FileResult(path, str(path.parent), parentlabels, labels)
if calculate_size:
fr.set_filesize(fp.stat().st_size)
return fr
def test_file_is_unpacked_by_extension(scan_environment):
fn = pathlib.Path("unpackers") / "gif" / "test.gif"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scanjob.set_scanenvironment(scan_environment)
scanjob.initialize()
unpacker = UnpackManager(scan_environment.unpackdirectory)
scanjob.prepare_for_unpacking()
scanjob.check_for_valid_extension(unpacker)
assert 'gif' in fileresult.labels
def test_carved_padding_file_has_correct_labels(scan_environment):
padding_file = _create_padding_file_in_unpack_directory(scan_environment)
fileresult = FileResult(None,
scan_environment.unpackdirectory / padding_file,
set())
fileresult.set_filesize(
(scan_environment.unpackdirectory / padding_file).stat().st_size)
scanjob = ScanJob(fileresult)
scanjob.set_scanenvironment(scan_environment)
scanjob.initialize()
unpacker = UnpackManager(scan_environment.unpackdirectory)
scanjob.prepare_for_unpacking()
scanjob.check_unscannable_file()
unpacker.append_unpacked_range(0, 5) # bytes [0:5) are unpacked
scanjob.carve_file_data(unpacker)
j = scan_environment.scanfilequeue.get()
assert j.fileresult.labels == set(['padding', 'synthesized'])
def unpack(self):
fns = ["sig1_first", "sig1_second"]
for fn in fns:
self._write_unpacked_file(fn)
return [
FileResult(self.fileresult, self.rel_unpack_dir / pathlib.Path(fn),
[]) for fn in fns
]
def create_unpackparser_for_path(scan_environment,
testdata_dir,
rel_testfile,
unpackparser,
offset,
data_unpack_dir=pathlib.Path('.'),
has_unpack_parent=False,
calculate_size=True):
"""Creates an unpackparser of type unpackparser to unpack the file
rel_testfile, starting at offset.
data_unpack_dir is the path of the directory to which any files are
extracted. The path is relative to the unpack root directory.
has_unpack_parent indicates if this file is unpacked from another file.
if True, rel_testfile is relative to the unpack root directory,
if False, rel_testfile is relative to the testdata directory.
calculate_size will calculate the size of the file. If the file does not
exist for some reason, this flag can be set to False. Default is
True.
"""
# self._copy_file_from_testdata(rel_testfile)
if has_unpack_parent:
parent = FileResult(None, rel_testfile.parent, set())
fileresult = FileResult(parent, rel_testfile, set())
else:
fileresult = FileResult(None, testdata_dir / rel_testfile, set())
if calculate_size:
path = scan_environment.get_unpack_path_for_fileresult(fileresult)
fileresult.set_filesize(path.stat().st_size)
p = unpackparser(fileresult, scan_environment, data_unpack_dir, offset)
return p
def test_dhcpv6sh_has_correct_labels(scan_environment):
# /home/tim/bang-test-scrap/bang-scan-wd8il1i5/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/lib/netifd/proto/dhcpv6.sh
fn = pathlib.Path("a/dhcpv6.sh")
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result = scan_environment.resultqueue.get()
assert result.labels == set(['text', 'script', 'shell'])
def test_process_css_file_has_correct_labels(scan_environment):
# /home/tim/bang-test-scrap/bang-scan-jucli3nm/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/www/luci-static/bootstrap/cascade.css
fn = pathlib.Path("a/cascade.css")
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result = scan_environment.resultqueue.get()
assert result.labels == set(['binary', 'css'])
def test_process_paddingfile_has_correct_labels(scan_environment):
padding_file = _create_padding_file_in_unpack_directory(scan_environment)
fileresult = FileResult(None,
scan_environment.unpackdirectory / padding_file,
set(['padding']))
fileresult.set_filesize(
(scan_environment.unpackdirectory / padding_file).stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result = scan_environment.resultqueue.get()
assert result.labels == set(['binary', 'padding'])
def test_openwrt_version_has_correct_labels(scan_environment):
# openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/etc/openwrt_version
fn = pathlib.Path("a/openwrt_version")
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as ex:
if ex.e.__class__ != QueueEmptyError:
raise ex
result = scan_environment.resultqueue.get()
assert result.labels == set(['text', 'base64', 'urlsafe'])
def test_gzip_unpacks_to_right_directory(scan_environment):
fn = pathlib.Path("a") / "hello.gz"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
fn_expected = pathlib.Path(fn.name + '-0x00000000-gzip-1') / 'hello'
assert result2.filename == fn_expected
def test_kernelconfig_is_processed(scan_environment):
# rel_testfile = pathlib.Path('unpackers') / 'kernelconfig' / 'kernelconfig'
rel_testfile = pathlib.Path(
'download') / 'system' / 'kernelconfig' / 'tiny.config'
abs_testfile = testdata_dir / rel_testfile
# TODO: FileResult asks for relative path
fileresult = FileResult(None, abs_testfile, set())
fileresult.set_filesize(abs_testfile.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result = scan_environment.resultqueue.get()
assert result.filename == abs_testfile
assert result.labels == set(['text', 'kernel configuration'])
def test_carved_data_is_extracted_from_file(scan_environment):
fn = pathlib.Path("unpackers") / "gif" / "test-prepend-random-data.gif"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scanjob.set_scanenvironment(scan_environment)
scanjob.initialize()
unpacker = UnpackManager(scan_environment.unpackdirectory)
scanjob.prepare_for_unpacking()
scanjob.check_for_valid_extension(unpacker)
scanjob.check_for_signatures(unpacker)
j = scan_environment.scanfilequeue.get()
scanjob.carve_file_data(unpacker)
j = scan_environment.scanfilequeue.get()
synthesized_name = pathlib.Path('.') / \
("%s-0x%08x-synthesized-1" % (fn.name,0)) / \
("unpacked-0x%x-0x%x" % (0,127))
assert j.fileresult.filename == synthesized_name
assertUnpackedPathExists(scan_environment, j.fileresult.filename)
def test_report_has_correct_path(scan_environment):
fn = pathlib.Path("a") / "hello.gz"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
unpack_report = result1.unpackedfiles[0]
fn_expected = pathlib.Path(fn.name + '-0x00000000-gzip-1') / 'hello'
assert unpack_report['unpackdirectory'] == fn_expected.parent
assert unpack_report['files'] == [fn_expected]
def test_double_gif_file_increases_name_counter(scan_environment):
fn = pathlib.Path("unpackers") / "gif" / "double.gif"
fn_abs = testdata_dir / fn
# TODO: FileResult asks for relative path
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
scan_environment.createjson = False
try:
processfile(MockDBConn(), MockDBCursor(), scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
result3 = scan_environment.resultqueue.get()
result_dir_nr2 = str(result3.filename.parent).split('-')[-1]
result_dir_nr3 = str(result3.filename.parent).split('-')[-1]
assert int(result_dir_nr2) == 1
assert int(result_dir_nr3) == 2
def test_file_with_extension_match_is_carved(scan_environment):
fn = pathlib.Path("unpackers") / "combined" / "double-gimpbrush.gbr"
fn_abs = testdata_dir / fn
fileresult = FileResult(None, fn_abs, set())
fileresult.set_filesize(fn_abs.stat().st_size)
scanjob = ScanJob(fileresult)
scan_environment.scanfilequeue.put(scanjob)
try:
processfile(scan_environment)
except QueueEmptyError:
pass
except ScanJobError as e:
if e.e.__class__ != QueueEmptyError:
raise e
assert len(scan_environment.resultqueue.queue) == 3
result1 = scan_environment.resultqueue.get()
result2 = scan_environment.resultqueue.get()
result3 = scan_environment.resultqueue.get()
assert result1.filename == fn_abs # parent file is absolute
assert result2.filename.name == 'unpacked.gimpbrush' # relative
assert result2.filename.parent.parent == pathlib.Path('.')
assert result3.filename.name == 'unpacked.gimpbrush' # relative
assert result3.filename.parent.parent == pathlib.Path('.')
def parse_and_unpack_success(self):
r = UnpackResults()
fr = FileResult(self.fileresult, self.get_carved_filename(), set())
r.set_unpacked_files([fr])
r.set_length(self.length)
return r
def _create_fileresult_for_file(self, child, parent, labels):
return FileResult(self._create_absolute_path_object(child), child,
self._create_absolute_path_object(parent), parent,
labels)
def create_tmp_fileresult(path_abs, content):
with open(path_abs, 'wb') as f:
f.write(content)
fileresult = FileResult(None, path_abs, set())
fileresult.set_filesize(path_abs.stat().st_size)
return fileresult