class TestUnpackerBase(unittest.TestCase):
def setUp(self):
config = ConfigParser()
self.ds_tmp_dir = TemporaryDirectory(prefix='faf_tests_')
config.add_section('data_storage')
config.set('data_storage', 'firmware_file_storage_directory', self.ds_tmp_dir.name)
config.add_section('unpack')
config.set('unpack', 'max_depth', '3')
config.set('unpack', 'whitelist', 'text/plain, image/png')
config.add_section('ExpertSettings')
self.unpacker = Unpacker(config=config, db_interface=DatabaseMock())
self.tmp_dir = TemporaryDirectory(prefix='faf_tests_')
self.test_fo = create_test_file_object()
def tearDown(self):
self.ds_tmp_dir.cleanup()
self.tmp_dir.cleanup()
gc.collect()
def check_unpacker_selection(self, mime_type, plugin_name, depth=0):
name = self.unpacker.get_unpacker(mime_type, depth)[1]
self.assertEqual(name, plugin_name, 'wrong unpacker plugin selected')
def check_unpacking_of_standard_unpack_set(self, in_file, additional_prefix_folder='', output=True):
files, meta_data = self.unpacker.extract_files_from_file(in_file, self.tmp_dir.name)
files = set(files)
self.assertEqual(len(files), 3, 'file number incorrect')
self.assertEqual(files, {
os.path.join(self.tmp_dir.name, additional_prefix_folder, 'testfile1'),
os.path.join(self.tmp_dir.name, additional_prefix_folder, 'testfile2'),
os.path.join(self.tmp_dir.name, additional_prefix_folder, 'generic folder/test file 3_.txt')
}, 'not all files found')
if output:
self.assertIn('output', meta_data)
return meta_data
def unpack_worker(self, worker_id):
unpacker = Unpacker(self.config, worker_id=worker_id, db_interface=self.db_interface)
while self.stop_condition.value == 0:
with suppress(Empty):
fo = self.in_queue.get(timeout=float(self.config['ExpertSettings']['block_delay']))
extracted_objects = unpacker.unpack(fo)
logging.debug('[worker {}] unpacking of {} complete: {} files extracted'.format(worker_id, fo.uid, len(extracted_objects)))
self.post_unpack(fo)
self.schedule_extracted_files(extracted_objects)
class TestUnpackerBase(unittest.TestCase):
def setUp(self):
self.config = ConfigParser()
self.ds_tmp_dir = TemporaryDirectory(prefix='fact_tests_')
self.tmp_dir = TemporaryDirectory(prefix='fact_tests_')
self.config.add_section('unpack')
self.config.set('unpack', 'data_folder', self.ds_tmp_dir.name)
self.config.set('unpack', 'blacklist', 'text/plain, image/png')
self.config.add_section('ExpertSettings')
self.config.set('ExpertSettings', 'header_overhead', '256')
self.config.set('ExpertSettings', 'unpack_threshold', '0.8')
self.unpacker = Unpacker(config=self.config)
os.makedirs(str(self.unpacker._report_folder), exist_ok=True) # pylint: disable=protected-access
os.makedirs(str(self.unpacker._file_folder), exist_ok=True) # pylint: disable=protected-access
self.test_file_path = Path(get_test_data_dir(),
'get_files_test/testfile1')
def tearDown(self):
self.ds_tmp_dir.cleanup()
self.tmp_dir.cleanup()
gc.collect()
def get_unpacker_meta(self):
return json.loads(
Path(self.unpacker._report_folder, 'meta.json').read_text()) # pylint: disable=protected-access
def check_unpacker_selection(self, mime_type, plugin_name):
name = self.unpacker.get_unpacker(mime_type)[1]
self.assertEqual(name, plugin_name, 'wrong unpacker plugin selected')
def check_unpacking_of_standard_unpack_set(self,
in_file: Union[Path, str],
additional_prefix_folder='',
output=True):
files, meta_data = self.unpacker.extract_files_from_file(
str(in_file), self.tmp_dir.name)
files = set(files)
self.assertEqual(len(files), 3, 'file number incorrect')
self.assertEqual(
files, {
os.path.join(self.tmp_dir.name, additional_prefix_folder,
'testfile1'),
os.path.join(self.tmp_dir.name, additional_prefix_folder,
'testfile2'),
os.path.join(self.tmp_dir.name, additional_prefix_folder,
'generic folder/test file 3_.txt')
}, 'not all files found')
if output:
self.assertIn('output', meta_data)
return meta_data
def setUp(self):
config = ConfigParser()
self.ds_tmp_dir = TemporaryDirectory(prefix='faf_tests_')
config.add_section('data_storage')
config.set('data_storage', 'firmware_file_storage_directory', self.ds_tmp_dir.name)
config.add_section('unpack')
config.set('unpack', 'max_depth', '3')
config.set('unpack', 'whitelist', 'text/plain, image/png')
config.add_section('ExpertSettings')
self.unpacker = Unpacker(config=config, db_interface=DatabaseMock())
self.tmp_dir = TemporaryDirectory(prefix='faf_tests_')
self.test_fo = create_test_file_object()
def setUp(self):
self.config = ConfigParser()
self.ds_tmp_dir = TemporaryDirectory(prefix='fact_tests_')
self.tmp_dir = TemporaryDirectory(prefix='fact_tests_')
self.config.add_section('unpack')
self.config.set('unpack', 'data_folder', self.ds_tmp_dir.name)
self.config.set('unpack', 'blacklist', 'text/plain, image/png')
self.config.add_section('ExpertSettings')
self.config.set('ExpertSettings', 'header_overhead', '256')
self.unpacker = Unpacker(config=self.config)
os.makedirs(str(self.unpacker._report_folder), exist_ok=True)
os.makedirs(str(self.unpacker._file_folder), exist_ok=True)
self.test_file_path = Path(get_test_data_dir(),
'get_files_test/testfile1')
def get_extracted_files(self, file_path1, file_path2):
"""
Recursively extract the content of both file paths and return a list of
files contained in each one
"""
file_path1 = pathlib.Path(file_path1).resolve()
file_path2 = pathlib.Path(file_path2).resolve()
config1 = deepcopy(self.config)
config1.update("data_folder", self.config.get("unpack",
"data_folder_1"))
config2 = deepcopy(self.config)
config2.update("data_folder", self.config.get("unpack",
"data_folder_2"))
# Create output "report" folder or fact extractor will crash
report_path = os.path.join(config1.get("unpack", "data_folder"),
"reports")
os.makedirs(report_path, exist_ok=True)
report_path = os.path.join(config2.get("unpack", "data_folder"),
"reports")
os.makedirs(report_path, exist_ok=True)
if self.config.extract:
unpacker1 = Unpacker(config=config1)
unpacker2 = Unpacker(config=config2)
files1 = self.extract(file_path1, unpacker1, config1)
files2 = self.extract(file_path2, unpacker2, config2)
data_folder_1 = "/tmp/extractor1"
data_folder_2 = "/tmp/extractor2"
else:
files1 = self.list_files(file_path1)
files2 = self.list_files(file_path2)
data_folder_1 = file_path1 if file_path1.is_dir(
) else file_path1.parent
data_folder_2 = file_path2 if file_path2.is_dir(
) else file_path2.parent
return files1, files2, data_folder_1, data_folder_2