def _extract_files_from_file_using_specific_unpacker(
self,
file_path: str,
tmp_dir: str,
selected_unpacker,
meta_data: dict = None) -> Tuple[List, Dict]:
unpack_function, name, version = selected_unpacker # TODO Refactor register method to directly use four parameters instead of three
if meta_data is None:
meta_data = {}
meta_data['plugin_used'] = name
meta_data['plugin_version'] = version
logging.debug('Try to unpack {} with {} plugin...'.format(
file_path, name))
try:
additional_meta = unpack_function(file_path, tmp_dir)
except Exception as e:
logging.debug('Unpacking of {} failed: {}: {}'.format(
file_path, type(e), str(e)))
additional_meta = {'error': '{}: {}'.format(type(e), str(e))}
if isinstance(additional_meta, dict):
meta_data.update(additional_meta)
self.change_owner_back_to_me(directory=tmp_dir)
meta_data['analysis_date'] = time()
return get_files_in_dir(tmp_dir), meta_data
def _extract_files_from_file_using_specific_unpacker(self, file_path: str, tmp_dir: str, selected_unpacker, meta_data: dict = None) -> Tuple[List, Dict]:
unpack_function, name, version = selected_unpacker # TODO Refactor register method to directly use four parameters instead of three
if meta_data is None:
meta_data = {}
meta_data['plugin_used'] = name
meta_data['plugin_version'] = version
logging.debug(f'Try to unpack {file_path} with {name} plugin...')
try:
additional_meta = unpack_function(file_path, tmp_dir)
except Exception as error:
logging.debug(f'Unpacking of {file_path} failed: {error}', exc_info=True)
additional_meta = {'error': f'{type(error)}: {str(error)}'}
if isinstance(additional_meta, dict):
meta_data.update(additional_meta)
self.change_owner_back_to_me(directory=tmp_dir)
meta_data['analysis_date'] = time()
out = get_files_in_dir(tmp_dir)
if self.exclude:
# Remove paths that should be ignored
excluded_count = len(out)
out = [f for f in out if not self._should_ignore(f)]
excluded_count -= len(out)
else:
excluded_count = 0
meta_data['number_of_excluded_files'] = excluded_count
return out, meta_data
def test_get_fact_bin_dir(self):
bin_dir = get_fact_bin_dir()
files_in_bin_dir = [
os.path.basename(f) for f in get_files_in_dir(bin_dir)
]
self.assertTrue(os.path.isdir(bin_dir))
self.assertIn('fact_extractor/bin', bin_dir)
self.assertIn('untrx', files_in_bin_dir)
def _get_list_of_files():
all_files = get_files_in_dir('.')
bin_files = [
any_file for any_file in all_files if any_file.endswith('a9s')
]
files = [('{}a9h'.format(bin_file[:len(bin_file) - 3]), bin_file)
for bin_file in bin_files]
return files
def _get_list_of_files(tmp_dir):
all_files = get_files_in_dir(tmp_dir)
bin_files = [
any_file for any_file in all_files if any_file.endswith('bin')
]
files = [('{}ini'.format(bin_file[:len(bin_file) - 3]), bin_file)
for bin_file in bin_files]
return files
def test_get_faf_bin_dir(self):
bin_dir = get_faf_bin_dir()
files_in_bin_dir = [
os.path.basename(f) for f in get_files_in_dir(bin_dir)
]
self.assertTrue(os.path.isdir(bin_dir))
self.assertIn('src/bin', bin_dir)
self.assertIn('passwords.txt', files_in_bin_dir)
def _create_joint_signature_file(directory, tmp_file):
all_signatures = list()
for signature_file in sorted(get_files_in_dir(directory)):
with open(signature_file, 'rb') as fd:
all_signatures.append(fd.read())
with open(tmp_file.name, 'wb') as fd:
fd.write(b'\x0a'.join(all_signatures))
def test_process_file():
tmp_dir = TemporaryDirectory()
re_organizer = ReOrganizer(out_dir=tmp_dir.name, testing=False)
re_organizer._process_file(TEST_FILE_PATH)
result = get_files_in_dir(tmp_dir.name)
assert len(result) == 1
assert re.search(
r'image\/png\/[0-9]{4}\/[0-9]{4}-[0-9]{2}-[0-9]{2}_small_image\.png',
result[0]) is not None
def _get_view_file_path(self, plugin_path):
plugin_path = get_parent_dir(get_dir_of_file(plugin_path))
view_files = get_files_in_dir(os.path.join(plugin_path, 'view'))
if len(view_files) < 1:
logging.debug('{}: No view available! Generic view will be used.'.format(self.NAME))
return None
elif len(view_files) > 1:
logging.warning('{}: Plug-in provides more than one view! \'{}\' is used!'.format(self.NAME, view_files[0]))
return view_files[0]
def test_get_files_in_dir(self):
test_dir_path = os.path.join(self.get_directory_of_current_file(),
"data")
result = get_files_in_dir(test_dir_path)
self.assertIn(os.path.join(test_dir_path, "read_test"), result,
"file in root folder not found")
self.assertIn(
os.path.join(test_dir_path, "test_folder/generic_test_file"),
result, "file in sub folder not found")
self.assertEqual(len(result), 6, "number of found files not correct")
def _extract_files_from_file_using_specific_unpacker(
self, file_path, tmp_dir, selected_unpacker, meta_data=None):
if meta_data is None:
meta_data = {}
meta_data['plugin_used'] = selected_unpacker[1]
meta_data['plugin_version'] = self._get_unpacker_version(
selected_unpacker)
logging.debug('[worker {}] Try to unpack {} with {} plugin...'.format(
self.worker_id, file_path, meta_data['plugin_used']))
try:
additional_meta = selected_unpacker[0](file_path, tmp_dir)
except Exception as e:
logging.debug('[worker {}] Unpacking of {} failed: {}: {}'.format(
self.worker_id, file_path,
sys.exc_info()[0].__name__, e))
additional_meta = {
'error': '{}: {}'.format(sys.exc_info()[0].__name__,
e.__str__())
}
if isinstance(additional_meta, dict):
meta_data.update(additional_meta)
self.change_owner_back_to_me(directory=tmp_dir)
meta_data['analysis_date'] = time()
return get_files_in_dir(tmp_dir), meta_data
def _remove_ini_files(tmp_dir):
for file_name in get_files_in_dir(tmp_dir):
if file_name.endswith('.ini'):
delete_file(file_name)
def get_signature_paths(sig_dir):
sig_files = get_files_in_dir(sig_dir)
signature_paths = make_dict_from_list(sig_files)
if len(signature_paths) == 0:
logging.error("no signature file found in {}".format(signature_paths))
return signature_paths
def check_meta_fields(self, sig_path):
sig_dir = sig_path
list_of_files = get_files_in_dir(sig_dir)
for file in list_of_files:
self.check_for_file(file)
return self.missing_meta_fields
def test_reorganize_files():
tmp_dir = TemporaryDirectory()
re_organizer = ReOrganizer(out_dir=tmp_dir.name, testing=False)
re_organizer.reorganize_files(TEST_DATA_DIR)
result = get_files_in_dir(tmp_dir.name)
assert len(result) == 5
def create_blacklist(input_dir, out_file):
input_files = get_files_in_dir(input_dir)
logging.info('{} files found'.format(len(input_files)))
blacklist_uids = _generate_blacklist_uids(input_files)
logging.info('Blacklisting {} unique files'.format(len(blacklist_uids)))
_store_blacklist(blacklist_uids, out_file)
def _remove_ini_files():
for file_name in get_files_in_dir('.'):
if file_name.endswith('.a9h'):
remove(file_name)
def test_get_faf_bin_dir(self):
bin_dir = get_faf_bin_dir()
files_in_bin_dir = [os.path.basename(f) for f in get_files_in_dir(bin_dir)]
self.assertTrue(os.path.isdir(bin_dir))
self.assertIn('src/bin', bin_dir)
self.assertIn('custommime.mgc', files_in_bin_dir)
def test_get_files_in_dir_error(self):
result = get_files_in_dir("/none_existing/dir")
self.assertEqual(result, [], "error result should be an empty list")
def reorganize_files(self, input_dir):
files = get_files_in_dir(input_dir)
logging.info('{} files found in {}'.format(len(files), input_dir))
for file in files:
self._process_file(file)
self._log_stats()
def _unpack_success(tmp_dir):
return len(get_files_in_dir(str(tmp_dir))) > 0
