def _extract_files_from_file_using_specific_unpacker( self, file_path: str, tmp_dir: str, selected_unpacker, meta_data: dict = None) -> Tuple[List, Dict]: unpack_function, name, version = selected_unpacker # TODO Refactor register method to directly use four parameters instead of three if meta_data is None: meta_data = {} meta_data['plugin_used'] = name meta_data['plugin_version'] = version logging.debug('Try to unpack {} with {} plugin...'.format( file_path, name)) try: additional_meta = unpack_function(file_path, tmp_dir) except Exception as e: logging.debug('Unpacking of {} failed: {}: {}'.format( file_path, type(e), str(e))) additional_meta = {'error': '{}: {}'.format(type(e), str(e))} if isinstance(additional_meta, dict): meta_data.update(additional_meta) self.change_owner_back_to_me(directory=tmp_dir) meta_data['analysis_date'] = time() return get_files_in_dir(tmp_dir), meta_data
def _extract_files_from_file_using_specific_unpacker(self, file_path: str, tmp_dir: str, selected_unpacker, meta_data: dict = None) -> Tuple[List, Dict]: unpack_function, name, version = selected_unpacker # TODO Refactor register method to directly use four parameters instead of three if meta_data is None: meta_data = {} meta_data['plugin_used'] = name meta_data['plugin_version'] = version logging.debug(f'Try to unpack {file_path} with {name} plugin...') try: additional_meta = unpack_function(file_path, tmp_dir) except Exception as error: logging.debug(f'Unpacking of {file_path} failed: {error}', exc_info=True) additional_meta = {'error': f'{type(error)}: {str(error)}'} if isinstance(additional_meta, dict): meta_data.update(additional_meta) self.change_owner_back_to_me(directory=tmp_dir) meta_data['analysis_date'] = time() out = get_files_in_dir(tmp_dir) if self.exclude: # Remove paths that should be ignored excluded_count = len(out) out = [f for f in out if not self._should_ignore(f)] excluded_count -= len(out) else: excluded_count = 0 meta_data['number_of_excluded_files'] = excluded_count return out, meta_data
def test_get_fact_bin_dir(self): bin_dir = get_fact_bin_dir() files_in_bin_dir = [ os.path.basename(f) for f in get_files_in_dir(bin_dir) ] self.assertTrue(os.path.isdir(bin_dir)) self.assertIn('fact_extractor/bin', bin_dir) self.assertIn('untrx', files_in_bin_dir)
def _get_list_of_files(): all_files = get_files_in_dir('.') bin_files = [ any_file for any_file in all_files if any_file.endswith('a9s') ] files = [('{}a9h'.format(bin_file[:len(bin_file) - 3]), bin_file) for bin_file in bin_files] return files
def _get_list_of_files(tmp_dir): all_files = get_files_in_dir(tmp_dir) bin_files = [ any_file for any_file in all_files if any_file.endswith('bin') ] files = [('{}ini'.format(bin_file[:len(bin_file) - 3]), bin_file) for bin_file in bin_files] return files
def test_get_faf_bin_dir(self): bin_dir = get_faf_bin_dir() files_in_bin_dir = [ os.path.basename(f) for f in get_files_in_dir(bin_dir) ] self.assertTrue(os.path.isdir(bin_dir)) self.assertIn('src/bin', bin_dir) self.assertIn('passwords.txt', files_in_bin_dir)
def _create_joint_signature_file(directory, tmp_file): all_signatures = list() for signature_file in sorted(get_files_in_dir(directory)): with open(signature_file, 'rb') as fd: all_signatures.append(fd.read()) with open(tmp_file.name, 'wb') as fd: fd.write(b'\x0a'.join(all_signatures))
def test_process_file(): tmp_dir = TemporaryDirectory() re_organizer = ReOrganizer(out_dir=tmp_dir.name, testing=False) re_organizer._process_file(TEST_FILE_PATH) result = get_files_in_dir(tmp_dir.name) assert len(result) == 1 assert re.search( r'image\/png\/[0-9]{4}\/[0-9]{4}-[0-9]{2}-[0-9]{2}_small_image\.png', result[0]) is not None
def _get_view_file_path(self, plugin_path): plugin_path = get_parent_dir(get_dir_of_file(plugin_path)) view_files = get_files_in_dir(os.path.join(plugin_path, 'view')) if len(view_files) < 1: logging.debug('{}: No view available! Generic view will be used.'.format(self.NAME)) return None elif len(view_files) > 1: logging.warning('{}: Plug-in provides more than one view! \'{}\' is used!'.format(self.NAME, view_files[0])) return view_files[0]
def test_get_files_in_dir(self): test_dir_path = os.path.join(self.get_directory_of_current_file(), "data") result = get_files_in_dir(test_dir_path) self.assertIn(os.path.join(test_dir_path, "read_test"), result, "file in root folder not found") self.assertIn( os.path.join(test_dir_path, "test_folder/generic_test_file"), result, "file in sub folder not found") self.assertEqual(len(result), 6, "number of found files not correct")
def _extract_files_from_file_using_specific_unpacker( self, file_path, tmp_dir, selected_unpacker, meta_data=None): if meta_data is None: meta_data = {} meta_data['plugin_used'] = selected_unpacker[1] meta_data['plugin_version'] = self._get_unpacker_version( selected_unpacker) logging.debug('[worker {}] Try to unpack {} with {} plugin...'.format( self.worker_id, file_path, meta_data['plugin_used'])) try: additional_meta = selected_unpacker[0](file_path, tmp_dir) except Exception as e: logging.debug('[worker {}] Unpacking of {} failed: {}: {}'.format( self.worker_id, file_path, sys.exc_info()[0].__name__, e)) additional_meta = { 'error': '{}: {}'.format(sys.exc_info()[0].__name__, e.__str__()) } if isinstance(additional_meta, dict): meta_data.update(additional_meta) self.change_owner_back_to_me(directory=tmp_dir) meta_data['analysis_date'] = time() return get_files_in_dir(tmp_dir), meta_data
def _remove_ini_files(tmp_dir): for file_name in get_files_in_dir(tmp_dir): if file_name.endswith('.ini'): delete_file(file_name)
def get_signature_paths(sig_dir): sig_files = get_files_in_dir(sig_dir) signature_paths = make_dict_from_list(sig_files) if len(signature_paths) == 0: logging.error("no signature file found in {}".format(signature_paths)) return signature_paths
def check_meta_fields(self, sig_path): sig_dir = sig_path list_of_files = get_files_in_dir(sig_dir) for file in list_of_files: self.check_for_file(file) return self.missing_meta_fields
def test_reorganize_files(): tmp_dir = TemporaryDirectory() re_organizer = ReOrganizer(out_dir=tmp_dir.name, testing=False) re_organizer.reorganize_files(TEST_DATA_DIR) result = get_files_in_dir(tmp_dir.name) assert len(result) == 5
def create_blacklist(input_dir, out_file): input_files = get_files_in_dir(input_dir) logging.info('{} files found'.format(len(input_files))) blacklist_uids = _generate_blacklist_uids(input_files) logging.info('Blacklisting {} unique files'.format(len(blacklist_uids))) _store_blacklist(blacklist_uids, out_file)
def _remove_ini_files(): for file_name in get_files_in_dir('.'): if file_name.endswith('.a9h'): remove(file_name)
def test_get_faf_bin_dir(self): bin_dir = get_faf_bin_dir() files_in_bin_dir = [os.path.basename(f) for f in get_files_in_dir(bin_dir)] self.assertTrue(os.path.isdir(bin_dir)) self.assertIn('src/bin', bin_dir) self.assertIn('custommime.mgc', files_in_bin_dir)
def test_get_files_in_dir_error(self): result = get_files_in_dir("/none_existing/dir") self.assertEqual(result, [], "error result should be an empty list")
def reorganize_files(self, input_dir): files = get_files_in_dir(input_dir) logging.info('{} files found in {}'.format(len(files), input_dir)) for file in files: self._process_file(file) self._log_stats()
def _unpack_success(tmp_dir): return len(get_files_in_dir(str(tmp_dir))) > 0