def test_featureless_file_is_unpacked(scan_environment): fn = pathlib.Path("unpackers") / "ihex" / "example.txt" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpacker = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_valid_extension(unpacker) assert fileresult.labels == set() scanjob.check_for_signatures(unpacker) assert fileresult.labels == set() assert fileresult.unpackedfiles == [] scanjob.carve_file_data(unpacker) assert fileresult.unpackedfiles == [] fileresult.labels.add('text') scanjob.check_entire_file(unpacker) assert len(fileresult.unpackedfiles) == 1 j = scan_environment.scanfilequeue.get() expected_extracted_fn = pathlib.Path('.') / \ ("%s-0x%08x-ihex-1" % (fn.name, 0)) / "unpacked-from-ihex" assert j.fileresult.filename == expected_extracted_fn assertUnpackedPathExists(scan_environment, j.fileresult.filename)
def test_file_with_signature_match_is_carved(scan_environment): fn = pathlib.Path("unpackers") / "combined" / "double-gimpbrush.bla" fn_abs = testdata_dir / fn fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e assert len(scan_environment.resultqueue.queue) == 3 result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() result3 = scan_environment.resultqueue.get() # unpack file at root has absolute path assert result1.filename == fn_abs assert result2.filename.name == 'unpacked.gimpbrush' assert result2.filename.parent.parent == pathlib.Path('.') assert result3.filename.name == 'unpacked.gimpbrush' assert result3.filename.parent.parent == pathlib.Path('.')
def test_file_without_features_is_carved(scan_environment): fn = pathlib.Path("unpackers") / "combined" / "kernelconfig-gif.bla" fn_abs = testdata_dir / fn fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e print(scan_environment.resultqueue.queue) assert len(scan_environment.resultqueue.queue) == 3 # assertlen(scan_environment.resultqueue.queue) == 4 result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() result3 = scan_environment.resultqueue.get() # result4 = scan_environment.resultqueue.get() # first result is for the file we queued and has an absolute path assert result1.filename == fn_abs # second result is the one matched by signature assert result2.filename.name == 'unpacked.gif' assert result2.filename.parent.parent == pathlib.Path('.') # third result is synthesized # gif_offset = 202554 gif_offset = result1.unpackedfiles[0]['offset'] assert result3.filename.name == \ 'unpacked-0x%x-0x%x' % (0,gif_offset-1) assert 'kernel configuration' in result3.labels
def initialize_scanjob_and_unpacker(scan_environment, fileresult): scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpacker = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() return scanjob, unpacker
def test_process_paddingfile_has_correct_labels(self): self._create_padding_file_in_directory() fileresult = self._create_fileresult_for_file(self.padding_file, self.parent_dir, set(['padding'])) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass result = self.result_queue.get() self.assertSetEqual(result.labels, set(['binary', 'padding']))
def test_process_css_file_has_correct_labels(self): # /home/tim/bang-test-scrap/bang-scan-jucli3nm/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/www/luci-static/bootstrap/cascade.css self._create_css_file_in_directory() fileresult = self._create_fileresult_for_file(self.css_file, self.parent_dir, set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass result = self.result_queue.get() self.assertSetEqual(result.labels, set(['text', 'css']))
def test_file_is_unpacked_by_extension(scan_environment): fn = pathlib.Path("unpackers") / "gif" / "test.gif" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpacker = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_valid_extension(unpacker) assert 'gif' in fileresult.labels
def test_carved_padding_file_has_correct_labels(self): self._create_padding_file_in_directory() fileresult = self._create_fileresult_for_file(self.padding_file, self.parent_dir, []) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(self.scan_environment) unpacker = Unpacker() scanjob.prepare_for_unpacking() scanjob.check_unscannable_file() unpacker.append_unpacked_range(0, 5) # bytes [0:5) are unpacked scanjob.carve_file_data(unpacker) j = self.scanfile_queue.get() self.assertSetEqual(j.fileresult.labels, set(['padding', 'synthesized']))
def test_process_css_file_has_correct_labels(self): # /home/tim/bang-test-scrap/bang-scan-jucli3nm/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/www/luci-static/bootstrap/cascade.css fn = pathlib.Path("a/cascade.css") self._copy_file_from_testdata(fn) fileresult = create_fileresult_for_path(self.unpackdir, fn, set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = self.result_queue.get() self.assertSetEqual(result.labels, set(['text', 'css']))
def test_file_unpack_signature_fail(scan_environment): fn = pathlib.Path("test.sig1") fileresult = create_tmp_fileresult( scan_environment.temporarydirectory / fn, b"A" * 70) scan_environment.set_unpackparsers([UnpackParserExtractSig1Fail]) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpack_manager = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_signatures(unpack_manager) assertUnpackedPathDoesNotExist(scan_environment, unpack_manager.get_data_unpack_directory()) assert fileresult.unpackedfiles == []
def test_gzip_unpacks_to_right_directory(self): fn = pathlib.Path("a/hello.gz") self._copy_file_from_testdata(fn) fileresult = create_fileresult_for_path(self.unpackdir, fn, set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result1 = self.result_queue.get() result2 = self.result_queue.get() self.assertEqual(str(result2.filename), str(fn) + '-gzip-1/hello')
def test_dhcpv6sh_has_correct_labels(self): # /home/tim/bang-test-scrap/bang-scan-wd8il1i5/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/lib/netifd/proto/dhcpv6.sh fn = pathlib.Path("a/dhcpv6.sh") self._copy_file_from_testdata(fn) fileresult = create_fileresult_for_path(self.unpackdir, fn) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = self.result_queue.get() self.assertSetEqual(result.labels, set(['text', 'script', 'shell']))
def test_carved_padding_file_has_correct_labels(scan_environment): padding_file = _create_padding_file_in_unpack_directory(scan_environment) fileresult = FileResult(None, scan_environment.unpackdirectory / padding_file, set()) fileresult.set_filesize( (scan_environment.unpackdirectory / padding_file).stat().st_size) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpacker = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_unscannable_file() unpacker.append_unpacked_range(0, 5) # bytes [0:5) are unpacked scanjob.carve_file_data(unpacker) j = scan_environment.scanfilequeue.get() assert j.fileresult.labels == set(['padding', 'synthesized'])
def test_dhcpv6sh_has_correct_labels(scan_environment): # /home/tim/bang-test-scrap/bang-scan-wd8il1i5/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/lib/netifd/proto/dhcpv6.sh fn = pathlib.Path("a/dhcpv6.sh") fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = scan_environment.resultqueue.get() assert result.labels == set(['text', 'script', 'shell'])
def test_openwrt_version_has_correct_labels(self): # openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/etc/openwrt_version fn = pathlib.Path("a/openwrt_version") self._copy_file_from_testdata(fn) fileresult = create_fileresult_for_path(self.unpackdir, fn, set()) # fileresult = self._create_fileresult_for_file(fn, os.path.dirname(fn), set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as ex: if ex.e.__class__ != QueueEmptyError: raise ex result = self.result_queue.get() self.assertSetEqual(result.labels, set(['text', 'base64', 'urlsafe']))
def test_process_paddingfile_has_correct_labels(scan_environment): padding_file = _create_padding_file_in_unpack_directory(scan_environment) fileresult = FileResult(None, scan_environment.unpackdirectory / padding_file, set(['padding'])) fileresult.set_filesize( (scan_environment.unpackdirectory / padding_file).stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = scan_environment.resultqueue.get() assert result.labels == set(['binary', 'padding'])
def test_process_css_file_has_correct_labels(scan_environment): # /home/tim/bang-test-scrap/bang-scan-jucli3nm/unpack/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/www/luci-static/bootstrap/cascade.css fn = pathlib.Path("a/cascade.css") fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = scan_environment.resultqueue.get() assert result.labels == set(['binary', 'css'])
def test_openwrt_version_has_correct_labels(scan_environment): # openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img.gz-gzip-1/openwrt-18.06.1-brcm2708-bcm2710-rpi-3-ext4-sysupgrade.img-ext2-1/etc/openwrt_version fn = pathlib.Path("a/openwrt_version") fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as ex: if ex.e.__class__ != QueueEmptyError: raise ex result = scan_environment.resultqueue.get() assert result.labels == set(['text', 'base64', 'urlsafe'])
def test_gzip_unpacks_to_right_directory(scan_environment): fn = pathlib.Path("a") / "hello.gz" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() fn_expected = pathlib.Path(fn.name + '-0x00000000-gzip-1') / 'hello' assert result2.filename == fn_expected
def test_file_unpack_signature_success(scan_environment): fn = pathlib.Path("test.sig1") fileresult = create_tmp_fileresult( scan_environment.temporarydirectory / fn, b"A" * 70) scan_environment.set_unpackparsers([UnpackParserExtractSig1]) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpack_manager = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_signatures(unpack_manager) unpack_report = fileresult.unpackedfiles[0] assert len(unpack_report['files']) == 2 fn1 = unpack_manager.get_data_unpack_directory() / "sig1_first" fn2 = unpack_manager.get_data_unpack_directory() / "sig1_second" assert unpack_report['files'][0] == fn1 assert unpack_report['files'][1] == fn2 assertUnpackedPathExists(scan_environment, unpack_report['files'][0]) assertUnpackedPathExists(scan_environment, unpack_report['files'][1])
def test_kernelconfig_is_processed(self): rel_testfile = pathlib.Path( 'unpackers') / 'kernelconfig' / 'kernelconfig' self._copy_file_from_testdata(rel_testfile) fileresult = create_fileresult_for_path(self.unpackdir, rel_testfile, set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = self.result_queue.get() self.assertEqual(result.filename, rel_testfile) self.assertSetEqual(result.labels, set(['text', 'kernel configuration']))
def test_report_has_correct_path(self): fn = pathlib.Path("a/hello.gz") self._copy_file_from_testdata(fn) fileresult = create_fileresult_for_path(self.unpackdir, fn, set()) scanjob = ScanJob(fileresult) self.scanfile_queue.put(scanjob) try: processfile(self.dbconn, self.dbcursor, self.scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result1 = self.result_queue.get() result2 = self.result_queue.get() unpack_report = result1.unpackedfiles[0] self.assertEqual(unpack_report['unpackdirectory'], str(fn) + '-0x00000000-gzip-1') self.assertEqual(unpack_report['files'], [str(fn) + '-0x00000000-gzip-1/hello'])
def test_kernelconfig_is_processed(scan_environment): # rel_testfile = pathlib.Path('unpackers') / 'kernelconfig' / 'kernelconfig' rel_testfile = pathlib.Path( 'download') / 'system' / 'kernelconfig' / 'tiny.config' abs_testfile = testdata_dir / rel_testfile # TODO: FileResult asks for relative path fileresult = FileResult(None, abs_testfile, set()) fileresult.set_filesize(abs_testfile.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result = scan_environment.resultqueue.get() assert result.filename == abs_testfile assert result.labels == set(['text', 'kernel configuration'])
def test_carved_data_is_extracted_from_file(scan_environment): fn = pathlib.Path("unpackers") / "gif" / "test-prepend-random-data.gif" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpacker = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_valid_extension(unpacker) scanjob.check_for_signatures(unpacker) j = scan_environment.scanfilequeue.get() scanjob.carve_file_data(unpacker) j = scan_environment.scanfilequeue.get() synthesized_name = pathlib.Path('.') / \ ("%s-0x%08x-synthesized-1" % (fn.name,0)) / \ ("unpacked-0x%x-0x%x" % (0,127)) assert j.fileresult.filename == synthesized_name assertUnpackedPathExists(scan_environment, j.fileresult.filename)
def test_report_has_correct_path(scan_environment): fn = pathlib.Path("a") / "hello.gz" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() unpack_report = result1.unpackedfiles[0] fn_expected = pathlib.Path(fn.name + '-0x00000000-gzip-1') / 'hello' assert unpack_report['unpackdirectory'] == fn_expected.parent assert unpack_report['files'] == [fn_expected]
def test_file_with_extension_match_is_carved(scan_environment): fn = pathlib.Path("unpackers") / "combined" / "double-gimpbrush.gbr" fn_abs = testdata_dir / fn fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) try: processfile(scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e assert len(scan_environment.resultqueue.queue) == 3 result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() result3 = scan_environment.resultqueue.get() assert result1.filename == fn_abs # parent file is absolute assert result2.filename.name == 'unpacked.gimpbrush' # relative assert result2.filename.parent.parent == pathlib.Path('.') assert result3.filename.name == 'unpacked.gimpbrush' # relative assert result3.filename.parent.parent == pathlib.Path('.')
def test_double_gif_file_increases_name_counter(scan_environment): fn = pathlib.Path("unpackers") / "gif" / "double.gif" fn_abs = testdata_dir / fn # TODO: FileResult asks for relative path fileresult = FileResult(None, fn_abs, set()) fileresult.set_filesize(fn_abs.stat().st_size) scanjob = ScanJob(fileresult) scan_environment.scanfilequeue.put(scanjob) scan_environment.createjson = False try: processfile(MockDBConn(), MockDBCursor(), scan_environment) except QueueEmptyError: pass except ScanJobError as e: if e.e.__class__ != QueueEmptyError: raise e result1 = scan_environment.resultqueue.get() result2 = scan_environment.resultqueue.get() result3 = scan_environment.resultqueue.get() result_dir_nr2 = str(result3.filename.parent).split('-')[-1] result_dir_nr3 = str(result3.filename.parent).split('-')[-1] assert int(result_dir_nr2) == 1 assert int(result_dir_nr3) == 2
def test_file_unpack_extension_carve(scan_environment): fn = pathlib.Path("test.ex1") fileresult = create_tmp_fileresult( scan_environment.temporarydirectory / fn, b"A" * 70) scan_environment.set_unpackparsers([UnpackParserExtractEx1Carve]) scanjob = ScanJob(fileresult) scanjob.set_scanenvironment(scan_environment) scanjob.initialize() unpack_manager = UnpackManager(scan_environment.unpackdirectory) scanjob.prepare_for_unpacking() scanjob.check_for_valid_extension(unpack_manager) unpack_report = fileresult.unpackedfiles[0] assert len(unpack_report['files']) == 3 fn1 = unpack_manager.get_data_unpack_directory() / "ex1_first" fn2 = unpack_manager.get_data_unpack_directory() / "ex1_second" fn3 = unpack_manager.get_data_unpack_directory( ) / "unpacked.ex1_extract_carve" assert unpack_report['files'][0] == fn1 assert unpack_report['files'][1] == fn2 assert unpack_report['files'][2] == fn3 assertUnpackedPathExists(scan_environment, unpack_report['files'][0]) assertUnpackedPathExists(scan_environment, unpack_report['files'][1]) assertUnpackedPathExists(scan_environment, unpack_report['files'][2])