def test_is_forced_update(self):
fo = MockFileObject()
assert self.scheduler._is_forced_update(fo) is False
fo.force_update = False
assert self.scheduler._is_forced_update(fo) is False
fo.force_update = True
assert self.scheduler._is_forced_update(fo) is True
def test_imphash(self):
file_path = os.path.join(TEST_DATA_DIR, 'ls')
result = self.analysis_plugin.process_object(
MockFileObject(
file_path=file_path)).processed_analysis[self.PLUGIN_NAME]
self.assertEqual(result['imphash'], 'd9eccd5f72564ac07601458b26040259',
'imphash not correct')
def test_find_artifacts(self):
fo = MockFileObject()
fo.processed_analysis['file_type'] = {'mime': 'text/plain'}
fo.virtual_file_path = {
1: ['some_uid|/home/user/project/.git/config',
'some_uid|/home/user/some_path/.pytest_cache/some_file',
'some_uid|/root/some_directory/some_more/.config/Code/User/settings.json',
'some_uid|/some_home/some_user/urandom/42/some_file.uvprojx',
'some_uid|some_more_uid|/this_home/this_dict/.zsh_history',
'some_uid|some_more_uid|/this_home/this_dict/.random_ambiguous_history',
'some_uid|home', 'some_uid|', 'some_uid|h654qf"§$%74672', 'some_uid|vuwreivh54r234/',
'some_uid|/vr4242fdsg4%%$']}
self.analysis_plugin.process_object(fo)
expected_result = sorted(['git_config', 'pytest_cache_directory', 'vscode_settings',
'keil_uvision_config', 'zsh_history', 'any_history'])
assert 'summary' in fo.processed_analysis[self.PLUGIN_NAME]
assert fo.processed_analysis[self.PLUGIN_NAME]['summary'] == expected_result
def test_get_blacklist_file_type_from_database(self):
def add_file_type_mock(_, fo):
fo.processed_analysis['file_type'] = {'mime': 'foo_type'}
file_object = MockFileObject()
file_object.processed_analysis.pop('file_type')
with mock_patch(self.sched, '_add_completed_analysis_results_to_file_object', add_file_type_mock):
result = self.sched._get_file_type_from_object_or_db(file_object)
assert result == 'foo_type'
def test_all_hashes(self):
self.fo = MockFileObject()
result = self.analysis_plugin.process_object(self.fo).processed_analysis[self.PLUGIN_NAME]
assert 'md5' in result, 'md5 not in result'
assert 'sha1' in result, 'sha1 not in result'
assert 'foo' not in result, 'foo in result but not available'
assert result['md5'] == '6f8db599de986fab7a21625b7916589c', 'hash not correct'
assert 'ssdeep' in result, 'ssdeep not in result'
assert 'imphash' in result, 'imphash not in result'
def test_all_hashes(self):
self.fo = MockFileObject()
result = self.analysis_plugin.process_object(
self.fo).processed_analysis[self.PLUGIN_NAME]
self.assertIn('md5', result, 'md5 not in result')
self.assertIn('sha1', result, 'sha1 not in result')
self.assertNotIn('foo', result, 'foo in result but not available')
self.assertEqual(result['md5'], '6f8db599de986fab7a21625b7916589c',
'hash not correct')
self.assertIn('ssdeep', result, 'ssdeep not in result')
self.assertIn('imphash', result, 'imphash not in result')
def test_find_path(self):
fo = MockFileObject()
fo.binary = (TEST_DATA_DIR / 'path_test_file').read_bytes()
fo.processed_analysis[self.PLUGIN_NAME] = {}
fo.processed_analysis['file_type'] = {'mime': 'application/x-executable'}
fo.virtual_file_path = {}
self.analysis_plugin.process_object(fo)
assert 'user_paths' in fo.processed_analysis[self.PLUGIN_NAME]
assert fo.processed_analysis[self.PLUGIN_NAME]['user_paths'] == ['/home/user/test/urandom', '/home/user/urandom']
assert 'www_path' in fo.processed_analysis[self.PLUGIN_NAME]
assert fo.processed_analysis[self.PLUGIN_NAME]['www_path'] == ['/var/www/tmp/me_']
assert 'root_path' in fo.processed_analysis[self.PLUGIN_NAME]
assert fo.processed_analysis[self.PLUGIN_NAME]['root_path'] == ['/root/user_name/this_directory']
assert 'summary' in fo.processed_analysis[self.PLUGIN_NAME]
assert fo.processed_analysis[self.PLUGIN_NAME]['summary'] == [
'/home/user/test/urandom', '/home/user/urandom', '/root/user_name/this_directory', '/var/www/tmp/me_'
]
class TestAnalysisSchedulerBlacklist:
test_plugin = 'test_plugin'
fo = MockFileObject()
class PluginMock:
def __init__(self, blacklist=None, whitelist=None):
if blacklist:
self.MIME_BLACKLIST = blacklist
if whitelist:
self.MIME_WHITELIST = whitelist
def shutdown(self):
pass
@classmethod
def setup_class(cls):
cls.init_patch = mock.patch(target='scheduler.Analysis.AnalysisScheduler.__init__', new=lambda *_: None)
cls.init_patch.start()
cls.sched = AnalysisScheduler()
cls.sched.analysis_plugins = {}
cls.plugin_list = ['no_deps', 'foo', 'bar']
cls.init_patch.stop()
def setup(self):
self.sched.config = get_config_for_testing()
def test_get_blacklist_and_whitelist_from_plugin(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_plugin('test_plugin')
assert (blacklist, whitelist) == (['foo'], ['bar'])
def test_get_blacklist_and_whitelist_from_plugin__missing_in_plugin(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_plugin('test_plugin')
assert whitelist == []
def test_get_blacklist_and_whitelist_from_config(self):
self._add_test_plugin_to_config()
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_config('test_plugin')
assert blacklist == ['type1', 'type2']
assert whitelist == []
def test_get_blacklist_and_whitelist__in_config_and_plugin(self):
self._add_test_plugin_to_config()
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist('test_plugin')
assert blacklist == ['type1', 'type2']
assert whitelist == []
def test_get_blacklist_and_whitelist__plugin_only(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist('test_plugin')
assert (blacklist, whitelist) == (['foo'], ['bar'])
def test_next_analysis_is_blacklisted__blacklisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=['blacklisted_type'])
self.fo.processed_analysis['file_type']['mime'] = 'blacklisted_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is True
def test_next_analysis_is_blacklisted__not_blacklisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=[])
self.fo.processed_analysis['file_type']['mime'] = 'not_blacklisted_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is False
def test_next_analysis_is_blacklisted__whitelisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(whitelist=['whitelisted_type'])
self.fo.processed_analysis['file_type']['mime'] = 'whitelisted_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is False
def test_next_analysis_is_blacklisted__not_whitelisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(whitelist=['some_other_type'])
self.fo.processed_analysis['file_type']['mime'] = 'not_whitelisted_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is True
def test_next_analysis_is_blacklisted__whitelist_precedes_blacklist(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=['test_type'], whitelist=['test_type'])
self.fo.processed_analysis['file_type']['mime'] = 'test_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is False
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=[], whitelist=['some_other_type'])
self.fo.processed_analysis['file_type']['mime'] = 'test_type'
blacklisted = self.sched._next_analysis_is_blacklisted(self.test_plugin, self.fo)
assert blacklisted is True
def test_get_blacklist_file_type_from_database(self):
def add_file_type_mock(_, fo):
fo.processed_analysis['file_type'] = {'mime': 'foo_type'}
file_object = MockFileObject()
file_object.processed_analysis.pop('file_type')
with mock_patch(self.sched, '_add_completed_analysis_results_to_file_object', add_file_type_mock):
result = self.sched._get_file_type_from_object_or_db(file_object)
assert result == 'foo_type'
def _add_test_plugin_to_config(self):
self.sched.config.add_section('test_plugin')
self.sched.config.set('test_plugin', 'mime_blacklist', 'type1, type2')
class TestAnalysisSchedulerBlacklist(AnalysisSchedulerTest):
test_plugin = 'test_plugin'
fo = MockFileObject()
class PluginMock:
def __init__(self, blacklist=None, whitelist=None):
if blacklist:
self.MIME_BLACKLIST = blacklist
if whitelist:
self.MIME_WHITELIST = whitelist
def shutdown(self):
pass
def test_get_blacklist_and_whitelist_from_plugin(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_plugin('test_plugin')
assert (blacklist, whitelist) == (['foo'], ['bar'])
def test_get_blacklist_and_whitelist_from_plugin__missing_in_plugin(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_plugin('test_plugin')
assert whitelist == []
def test_get_blacklist_and_whitelist_from_config(self):
self._add_test_plugin_to_config()
blacklist, whitelist = self.sched._get_blacklist_and_whitelist_from_config('test_plugin')
assert blacklist == ['type1', 'type2']
assert whitelist == []
def test_get_blacklist_and_whitelist__in_config_and_plugin(self):
self._add_test_plugin_to_config()
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist('test_plugin')
assert blacklist == ['type1', 'type2']
assert whitelist == []
def test_get_blacklist_and_whitelist__plugin_only(self):
self.sched.analysis_plugins['test_plugin'] = self.PluginMock(['foo'], ['bar'])
blacklist, whitelist = self.sched._get_blacklist_and_whitelist('test_plugin')
assert (blacklist, whitelist) == (['foo'], ['bar'])
def test_next_analysis_is_not_blacklisted__blacklisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=['blacklisted_type'])
self.fo.processed_analysis['file_type']['mime'] = 'blacklisted_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is False
def test_next_analysis_is_not_blacklisted__not_blacklisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=[])
self.fo.processed_analysis['file_type']['mime'] = 'not_blacklisted_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is True
def test_next_analysis_is_not_blacklisted__whitelisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(whitelist=['whitelisted_type'])
self.fo.processed_analysis['file_type']['mime'] = 'whitelisted_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is True
def test_next_analysis_is_not_blacklisted__not_whitelisted(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(whitelist=['some_other_type'])
self.fo.processed_analysis['file_type']['mime'] = 'not_whitelisted_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is False
def test_next_analysis_is_not_blacklisted__whitelist_precedes_blacklist(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=['test_type'], whitelist=['test_type'])
self.fo.processed_analysis['file_type']['mime'] = 'test_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is True
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=[], whitelist=['some_other_type'])
self.fo.processed_analysis['file_type']['mime'] = 'test_type'
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is False
def test_next_analysis_is_not_blacklisted__mime_missing(self):
self.sched.analysis_plugins[self.test_plugin] = self.PluginMock(blacklist=['test_type'], whitelist=['test_type'])
self.fo.processed_analysis['file_type'].pop('mime')
self.fo.scheduled_analysis = []
self.fo.analysis_dependency = set()
not_blacklisted = self.sched._next_analysis_is_not_blacklisted(self.test_plugin, self.fo)
assert not_blacklisted is False
assert 'file_type' in self.fo.analysis_dependency
assert self.fo.scheduled_analysis == [self.test_plugin, 'file_type']
def test_start_or_skip_analysis(self):
self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar')
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['file_hashes']
test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'}
self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw)
test_fw = self.tmp_queue.get(timeout=10)
assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis
assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def _add_test_plugin_to_config(self):
self.sched.config.add_section('test_plugin')
self.sched.config.set('test_plugin', 'mime_blacklist', 'type1, type2')
def test_imphash(self):
file_path = os.path.join(TEST_DATA_DIR, 'ls')
result = self.analysis_plugin.process_object(MockFileObject(file_path=file_path)).processed_analysis[self.PLUGIN_NAME]
assert isinstance(result['imphash'], str), 'imphash should be a string'
assert len(result['imphash']) == 32, 'imphash does not look like an md5'