class TestInterComListener(unittest.TestCase):
def setUp(self):
config = get_config_for_testing(temp_dir=TMP_DIR)
self.mongo_server = MongoMgr(config=config)
self.generic_listener = InterComListener(config=config)
def tearDown(self):
for item in self.generic_listener.connections.keys():
self.generic_listener.client.drop_database(
self.generic_listener.connections[item]['name'])
self.generic_listener.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def check_file(self, binary):
self.generic_listener.connections[
self.generic_listener.CONNECTION_TYPE]['fs'].put(
pickle.dumps(binary))
task = self.generic_listener.get_next_task()
self.assertEqual(task, binary)
another_task = self.generic_listener.get_next_task()
self.assertIsNone(another_task, 'task not deleted')
def test_small_file(self):
self.check_file(b'this is a test')
def test_big_file(self):
large_test_data = generate_random_data(size=BSON_MAX_FILE_SIZE + 1024)
self.check_file(large_test_data)
class TestWorkloadStatistic(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.workload_stat = WorkLoadStatistic(config=self.config,
component='test')
self.frontend_db_interface = StatisticDbViewer(config=self.config)
def tearDown(self):
self.frontend_db_interface.shutdown()
self.workload_stat.shutdown()
clean_test_database(self.config, get_database_names(self.config))
self.mongo_server.shutdown()
gc.collect()
def test_update_workload_statistic(self):
self.workload_stat.update()
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['name'], 'test', 'name not set')
self.assertAlmostEqual(time(),
result['last_update'],
msg='timestamp not valid',
delta=100)
self.assertIsInstance(result['platform'], dict,
'platfom is not a dict')
self.assertIsInstance(result['system'], dict, 'system is not a dict')
class TestInterComBackEndScheduler(unittest.TestCase):
def setUp(self):
config = get_config_for_testing(TMP_DIR)
self.test_queue = Queue()
self.interface = InterComBackEndBinding(
config=config,
testing=True,
analysis_service=AnalysisServiceMock(),
compare_service=ServiceMock(self.test_queue),
unpacking_service=ServiceMock(self.test_queue))
self.interface.WAIT_TIME = 2
self.db = MongoMgr(config=config)
def tearDown(self):
self.interface.shutdown()
self.test_queue.close()
self.db.shutdown()
TMP_DIR.cleanup()
gc.collect()
def test_backend_worker(self):
service = ServiceMock(self.test_queue)
self.interface._start_listener(CommunicationBackendMock,
service.add_task) # pylint: disable=protected-access
result = self.test_queue.get(timeout=5)
self.assertEqual(result, 'test_task', 'task not received correctly')
def test_all_listeners_started(self):
self.interface.startup()
sleep(2)
self.assertEqual(len(self.interface.process_list), NUMBER_OF_LISTENERS,
'Not all listeners started')
class TestTagPropagation(unittest.TestCase):
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.analysis_finished_event = Event()
self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048'
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._tagging_scheduler = TaggingDaemon(
analysis_scheduler=self._analysis_scheduler)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_object(fw_object)
if fw_object.uid == self.uid_of_key_file:
self.analysis_finished_event.set()
def _wait_for_empty_tag_queue(self):
while not self._analysis_scheduler.tag_queue.empty():
sleep(0.1)
def tearDown(self):
self._unpack_scheduler.shutdown()
self._tagging_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_run_analysis_with_tag(self):
test_fw = Firmware(
file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(
self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material'][
'tags'], 'no tags set in analysis'
self._wait_for_empty_tag_queue()
processed_fw = self.backend_interface.get_object(
test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material'][
'private_key_inside']
def binary_service():
with TemporaryDirectory(prefix='fact_test_') as tmp_dir:
config = get_config_for_testing(temp_dir=tmp_dir)
mongo_server = MongoMgr(config=config)
_init_test_data(config, tmp_dir)
yield BinaryService(config=config)
mongo_server.shutdown()
gc.collect()
def main(command_line_options=sys.argv):
_, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION, command_line_options=command_line_options)
logging.info('Trying to start Mongo Server and initializing users...')
mongo_manger = MongoMgr(config=config, auth=False)
mongo_manger.init_users()
mongo_manger.shutdown()
return 0
class FactDb(FactBase):
PROGRAM_NAME = 'FACT DB-Service'
PROGRAM_DESCRIPTION = 'Firmware Analysis and Compare Tool (FACT) DB-Service'
COMPONENT = 'database'
def __init__(self):
_, config = program_setup(self.PROGRAM_NAME, self.PROGRAM_DESCRIPTION)
self.mongo_server = MongoMgr(config=config)
super().__init__()
def shutdown(self):
super().shutdown()
self.mongo_server.shutdown()
class RestTestBase(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing(TMP_DIR)
self.mongo_mgr = MongoMgr(self.config)
self.frontend = WebFrontEnd(config=self.config)
self.frontend.app.config['TESTING'] = True
self.test_client = self.frontend.app.test_client()
def tearDown(self):
clean_test_database(self.config, get_database_names(self.config))
self.mongo_mgr.shutdown()
gc.collect()
def test_view_sync_interface():
mongo_server = MongoMgr(config=CONFIG)
view_update_service = ViewUpdater(config=CONFIG)
view_update_service.update_view('test', TEST_DATA)
view_update_service.shutdown()
view_read_service = ViewReader(config=CONFIG)
assert view_read_service.get_view('none_existing') is None
assert view_read_service.get_view('test') == TEST_DATA
view_read_service.shutdown()
mongo_server.shutdown()
gc.collect()
def main(command_line_options=sys.argv):
_, config = program_setup(PROGRAM_NAME,
PROGRAM_DESCRIPTION,
command_line_options=command_line_options)
logging.info('Trying to start Mongo Server and initializing users...')
mongo_manger = MongoMgr(config=config, auth=False)
db_service_frontend_editing = FrontendEditingDbInterface(config)
convert_comments_to_new_format(db_service_frontend_editing)
convert_release_dates_to_date_object_format(db_service_frontend_editing)
add_parent_firmware_list_to_file_object(db_service_frontend_editing)
mongo_manger.shutdown()
return 0
def main(command_line_options=sys.argv):
args, config = program_setup(PROGRAM_NAME,
PROGRAM_DESCRIPTION,
command_line_options=command_line_options)
logging.info('Try to start Mongo Server...')
mongo_server = MongoMgr(config=config)
updater = StatisticUpdater(config=config)
updater.update_all_stats()
updater.shutdown()
if args.testing:
logging.info('Stopping Mongo Server...')
mongo_server.shutdown()
return 0
def get_intercom_for_testing():
with TemporaryDirectory(prefix='fact_test_') as tmp_dir:
config = get_config_for_testing(tmp_dir)
test_queue = Queue()
interface = InterComBackEndBinding(
config=config,
testing=True,
analysis_service=AnalysisServiceMock(),
compare_service=ServiceMock(test_queue),
unpacking_service=ServiceMock(test_queue))
interface.WAIT_TIME = 2
db = MongoMgr(config=config)
yield interface
interface.shutdown()
test_queue.close()
db.shutdown()
gc.collect()
class TestBinaryService(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
def _init_test_data(self):
self.backend_db_interface = BackEndDbInterface(config=self.config)
self.backend_db_interface.add_firmware(TEST_FW)
self.backend_db_interface.shutdown()
def tearDown(self):
self.mongo_server.shutdown()
gc.collect()
def test_get_binary_and_file_name(self):
binary, file_name = self.binary_service.get_binary_and_file_name(
TEST_FW.uid)
self.assertEqual(file_name, TEST_FW.file_name, 'file_name not correct')
self.assertEqual(binary, TEST_FW.binary, 'invalid result not correct')
def test_get_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_binary_and_file_name(
'invalid_uid')
self.assertIsNone(binary, 'should be none')
self.assertIsNone(file_name, 'should be none')
def test_get_repacked_binary_and_file_name(self):
tar, file_name = self.binary_service.get_repacked_binary_and_file_name(
TEST_FW.uid)
self.assertEqual(file_name, '{}.tar.gz'.format(TEST_FW.file_name),
'file_name not correct')
file_type = magic.from_buffer(tar, mime=False)
assert 'gzip compressed data' in file_type, 'Result is not an tar.gz file'
def test_get_repacked_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_repacked_binary_and_file_name(
'invalid_uid')
self.assertIsNone(binary, 'should be none')
self.assertIsNone(file_name, 'should be none')
def main(command_line_options=None):
command_line_options = sys.argv if command_line_options is None else command_line_options
args, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION, command_line_options=command_line_options)
logging.info('Try to start Mongo Server...')
mongo_server = MongoMgr(config=config)
logging.info('updating data... this may take several hours depending on the size of your database')
start_time = time()
return_code = _create_variety_data(config)
process_time = time() - start_time
logging.info('generation time: {}'.format(time_format(process_time)))
if args.testing:
logging.info('Stopping Mongo Server...')
mongo_server.shutdown()
return return_code
class TestBinaryService:
def setup(self):
self.tmp_dir = TemporaryDirectory()
self.config = get_config_for_testing(temp_dir=self.tmp_dir)
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
def _init_test_data(self):
self.backend_db_interface = BackEndDbInterface(config=self.config)
self.backend_db_interface.add_firmware(TEST_FW)
store_binary_on_file_system(self.tmp_dir.name, TEST_FW)
self.backend_db_interface.shutdown()
def teardown(self):
self.tmp_dir.cleanup()
self.mongo_server.shutdown()
gc.collect()
def test_get_binary_and_file_name(self):
binary, file_name = self.binary_service.get_binary_and_file_name(TEST_FW.uid)
assert file_name == TEST_FW.file_name, 'file_name not correct'
assert binary == TEST_FW.binary, 'invalid result not correct'
def test_get_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_binary_and_file_name('invalid_uid')
assert binary is None, 'should be none'
assert file_name is None, 'should be none'
def test_get_repacked_binary_and_file_name(self):
tar, file_name = self.binary_service.get_repacked_binary_and_file_name(TEST_FW.uid)
assert file_name == '{}.tar.gz'.format(TEST_FW.file_name), 'file_name not correct'
file_type = magic.from_buffer(tar, mime=False)
assert 'gzip compressed data' in file_type, 'Result is not an tar.gz file'
def test_get_repacked_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_repacked_binary_and_file_name('invalid_uid')
assert binary is None, 'should be none'
assert file_name is None, 'should be none'
class TestSummary(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
def tearDown(self):
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.db_interface_backend.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
def create_and_add_test_fimrware_and_file_object(self):
self.test_fw = create_test_firmware()
self.test_fo = create_test_file_object()
self.test_fw.add_included_file(self.test_fo)
self.db_interface_backend.add_object(self.test_fw)
self.db_interface_backend.add_object(self.test_fo)
def test_get_set_of_all_included_files(self):
self.create_and_add_test_fimrware_and_file_object()
result_set_fo = self.db_interface.get_set_of_all_included_files(
self.test_fo)
self.assertIsInstance(result_set_fo, set, 'result is not a set')
self.assertEqual(len(result_set_fo), 1, 'number of files not correct')
self.assertIn(self.test_fo.uid, result_set_fo,
'object not in its own result set')
result_set_fw = self.db_interface.get_set_of_all_included_files(
self.test_fw)
self.assertEqual(len(result_set_fw), 2, 'number of files not correct')
self.assertIn(self.test_fo.uid, result_set_fw,
'test file not in result set firmware')
self.assertIn(self.test_fw.uid, result_set_fw,
'fw not in result set firmware')
def test_get_uids_of_all_included_files(self):
def add_test_file_to_db(uid, parent_uids: Set[str]):
test_fo = create_test_file_object()
test_fo.parent_firmware_uids = parent_uids
test_fo.uid = uid
self.db_interface_backend.add_object(test_fo)
add_test_file_to_db('uid1', {'foo'})
add_test_file_to_db('uid2', {'foo', 'bar'})
add_test_file_to_db('uid3', {'bar'})
result = self.db_interface.get_uids_of_all_included_files('foo')
assert result == {'uid1', 'uid2'}
assert self.db_interface.get_uids_of_all_included_files(
'uid not in db') == set()
def test_get_summary(self):
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface.get_summary(self.test_fw, 'dummy')
self.assertIsInstance(result_sum, dict, 'summary is not a dict')
self.assertIn('sum a', result_sum, 'summary entry of parent missing')
self.assertIn(self.test_fw.uid, result_sum['sum a'],
'origin (parent) missing in parent summary entry')
self.assertIn(self.test_fo.uid, result_sum['sum a'],
'origin (child) missing in parent summary entry')
self.assertNotIn(self.test_fo.uid, result_sum['fw exclusive sum a'],
'child as origin but should not be')
self.assertIn('file exclusive sum b', result_sum,
'file exclusive summary missing')
self.assertIn(self.test_fo.uid, result_sum['file exclusive sum b'],
'origin of file exclusive missing')
self.assertNotIn(self.test_fw.uid, result_sum['file exclusive sum b'],
'parent as origin but should not be')
def test_collect_summary(self):
self.create_and_add_test_fimrware_and_file_object()
fo_list = [self.test_fo.uid]
result_sum = self.db_interface._collect_summary(fo_list, 'dummy')
assert all(
item in result_sum
for item in self.test_fo.processed_analysis['dummy']['summary'])
assert all(value == [self.test_fo.uid]
for value in result_sum.values())
def test_get_summary_of_one_error_handling(self):
result_sum = self.db_interface._get_summary_of_one(None, 'foo')
self.assertEqual(result_sum, {},
'None object should result in empty dict')
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface._get_summary_of_one(
self.test_fw, 'none_existing_analysis')
self.assertEqual(result_sum, {},
'analysis not existend should lead to empty dict')
def test_update_summary(self):
orig = {'a': ['a']}
update = {'a': ['aa'], 'b': ['aa']}
result = self.db_interface._update_summary(orig, update)
self.assertIn('a', result)
self.assertIn('b', result)
self.assertIn('a', result['a'])
self.assertIn('aa', result['a'])
self.assertIn('aa', result['b'])
class TestCompare(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def tearDown(self):
self.db_interface_compare.shutdown()
self.db_interface_admin.shutdown()
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.mongo_server.shutdown()
gc.collect()
def _create_compare_dict(self):
comp_dict = {
'general': {
'hid': {
self.fw_one.get_uid(): 'foo',
self.fw_two.get_uid(): 'bar'
}
},
'plugins': {}
}
comp_dict['general']['virtual_file_path'] = {
self.fw_one.get_uid(): 'dev_one_name',
self.fw_two.get_uid(): 'dev_two_name'
}
return comp_dict
def test_add_and_get_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
retrieved = self.db_interface_compare.get_compare_result(
'{};{}'.format(self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertEqual(
retrieved['general']['virtual_file_path'][self.fw_one.get_uid()],
'dev_one_name', 'content of retrieval not correct')
def test_get_not_existing_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
result = self.db_interface_compare.get_compare_result('{};{}'.format(
self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertIsNone(result, 'result not none')
def test_calculate_compare_result_id(self):
comp_id = self.db_interface_compare._calculate_compare_result_id(
self.compare_dict)
self.assertEqual(
comp_id, '{};{}'.format(self.fw_one.get_uid(),
self.fw_two.get_uid()))
def test_object_existence_quick_check(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.assertIsNone(
self.db_interface_compare.object_existence_quick_check(
self.fw_one.get_uid()), 'existing_object not found')
self.assertEqual(
self.db_interface_compare.object_existence_quick_check(
'{};none_existing_object'.format(self.fw_one.get_uid())),
'none_existing_object not found in database',
'error message not correct')
def test_get_compare_result_of_none_existing_uid(self):
self.db_interface_backend.add_firmware(self.fw_one)
result = self.db_interface_compare.get_compare_result(
'{};none_existing_uid'.format(self.fw_one.get_uid()))
self.assertEqual(result, 'none_existing_uid not found in database',
'no result not found error')
def test_get_latest_comparisons(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
before = time()
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
for id, hids, submission_date in result:
self.assertIn(self.fw_one.get_uid(), hids)
self.assertIn(self.fw_two.get_uid(), hids)
self.assertIn(self.fw_one.get_uid(), id)
self.assertIn(self.fw_two.get_uid(), id)
self.assertTrue(before <= submission_date <= time())
def test_get_latest_comparisons_removed_firmware(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertNotEqual(result, [], 'A compare result should be available')
self.db_interface_admin.delete_firmware(self.fw_two.uid)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertEqual(result, [], 'No compare result should be available')
def test_get_total_number_of_results(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
number = self.db_interface_compare.get_total_number_of_results()
self.assertEqual(number, 1, 'no compare result found in database')
class TestStorageDbInterfaceFrontendEditing(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_frontend_editing = FrontendEditingDbInterface(
config=self._config)
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
def tearDown(self):
self.db_frontend_editing.shutdown()
self.db_frontend_interface.shutdown()
self.db_backend_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_backend_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def test_add_comment(self):
test_fw = create_test_firmware()
self.db_backend_interface.add_object(test_fw)
comment, author, uid, time = 'this is a test comment!', 'author', test_fw.get_uid(
), 1234567890
self.db_frontend_editing.add_comment_to_object(uid, comment, author,
time)
test_fw = self.db_backend_interface.get_object(uid)
self.assertEqual(test_fw.comments[0], {
'time': str(time),
'author': author,
'comment': comment
})
def test_get_latest_comments(self):
comments = [{
'time': '1234567890',
'author': 'author1',
'comment': 'test comment'
}, {
'time': '1234567899',
'author': 'author2',
'comment': 'test comment2'
}]
test_fw = self._add_test_fw_with_comments_to_db()
latest_comments = self.db_frontend_interface.get_latest_comments()
comments.sort(key=lambda x: x['time'], reverse=True)
for i in range(len(comments)):
time, author, comment, uid = comments[i]['time'], comments[i][
'author'], comments[i]['comment'], test_fw.get_uid()
self.assertEqual(latest_comments[i]['time'], time)
self.assertEqual(latest_comments[i]['author'], author)
self.assertEqual(latest_comments[i]['comment'], comment)
self.assertEqual(latest_comments[i]['uid'], uid)
def test_remove_element_from_array_in_field(self):
test_fw = self._add_test_fw_with_comments_to_db()
retrieved_fw = self.db_backend_interface.get_object(test_fw.get_uid())
self.assertEqual(len(retrieved_fw.comments), 2,
'comments were not saved correctly')
self.db_frontend_editing.remove_element_from_array_in_field(
test_fw.get_uid(), 'comments', {'time': '1234567899'})
retrieved_fw = self.db_backend_interface.get_object(test_fw.get_uid())
self.assertEqual(len(retrieved_fw.comments), 1,
'comment was not deleted')
def test_delete_comment(self):
test_fw = self._add_test_fw_with_comments_to_db()
retrieved_fw = self.db_backend_interface.get_object(test_fw.get_uid())
self.assertEqual(len(retrieved_fw.comments), 2,
'comments were not saved correctly')
self.db_frontend_editing.delete_comment(test_fw.get_uid(),
'1234567899')
retrieved_fw = self.db_backend_interface.get_object(test_fw.get_uid())
self.assertEqual(len(retrieved_fw.comments), 1,
'comment was not deleted')
def _add_test_fw_with_comments_to_db(self):
test_fw = create_test_firmware()
comments = [{
'time': '1234567890',
'author': 'author1',
'comment': 'test comment'
}, {
'time': '1234567899',
'author': 'author2',
'comment': 'test comment2'
}]
test_fw.comments.extend(comments)
self.db_backend_interface.add_object(test_fw)
return test_fw
def test_server(test_config):
mongo = MongoMgr(test_config)
clean_test_database(test_config, get_database_names(test_config))
yield None
clean_test_database(test_config, get_database_names(test_config))
mongo.shutdown()
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_object(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value > 7:
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=10)
compare_id = unify_string_list(';'.join([fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str), 'compare result should exist')
self.assertEqual(result['plugins']['Software'], self._expected_result()['Software'])
self.assertCountEqual(result['plugins']['File_Coverage']['exclusive_files'], self._expected_result()['File_Coverage']['exclusive_files'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'exclusive_files': {
'418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787': [],
'd38970f8c5153d1041810d0908292bc8df21e7fd88aab211a8fb96c54afe6b01_319': [],
'collapse': False
},
'files_in_common': {
'all': [
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'289b5a050a83837f192d7129e4c4e02570b94b4924e50159fad5ed1067cfbfeb_20',
'd558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62'
],
'collapse': False
},
'similar_files': {}
},
'Software': {
'Compare Skipped': {
'all': 'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
class TestMongoInterface(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self._config.set('data_storage', 'report_threshold', '32')
self._config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def tearDown(self):
self.db_interface_backend.client.drop_database(self._config.get('data_storage', 'main_database'))
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(self._config.get('data_storage', 'sanitize_database'))
self.db_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def _get_all_firmware_uids(self):
uid_list = []
tmp = self.db_interface.firmwares.find()
for item in tmp:
uid_list.append(item['_id'])
return uid_list
def test_existence_quick_check(self):
self.assertFalse(self.db_interface.existence_quick_check('none_existing'), 'none existing firmware found')
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertTrue(self.db_interface.existence_quick_check(self.test_firmware.get_uid()), 'existing firmware not found')
self.db_interface_backend.add_file_object(self.test_fo)
self.assertTrue(self.db_interface.existence_quick_check(self.test_fo.get_uid()), 'existing file not found')
def test_get_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fobject = self.db_interface.get_firmware(self.test_firmware.get_uid())
self.assertEqual(fobject.vendor, 'test_vendor')
self.assertEqual(fobject.device_name, 'test_router')
def test_get_object(self):
fo = self.db_interface.get_object(self.test_firmware.get_uid())
self.assertIsNone(fo, 'found something but there is nothing in the database')
self.db_interface_backend.add_firmware(self.test_firmware)
fo = self.db_interface.get_object(self.test_firmware.get_uid())
self.assertIsInstance(fo, Firmware, 'firmware has wrong type')
self.assertEqual(fo.device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo = self.db_interface.get_object(test_file.get_uid())
self.assertIsInstance(fo, FileObject, 'file object has wrong type')
def test_get_complete_object_including_all_summaries(self):
self.db_interface_backend.report_threshold = 1024
test_file = create_test_file_object()
self.test_firmware.add_included_file(test_file)
self.db_interface_backend.add_firmware(self.test_firmware)
self.db_interface_backend.add_file_object(test_file)
tmp = self.db_interface.get_complete_object_including_all_summaries(self.test_firmware.get_uid())
self.assertIsInstance(tmp, Firmware, 'wrong type')
self.assertIn('summary', tmp.processed_analysis['dummy'].keys(), 'summary not found in processed analysis')
self.assertIn('sum a', tmp.processed_analysis['dummy']['summary'], 'summary of original file not included')
self.assertIn('file exclusive sum b', tmp.processed_analysis['dummy']['summary'], 'summary of included file not found')
def test_sanitize_analysis(self):
short_dict = {'stub_plugin': {'result': 0}}
long_dict = {'stub_plugin': {'result': 10000000000, 'misc': 'Bananarama', 'summary': []}}
self.test_firmware.processed_analysis = short_dict
sanitized_dict = self.db_interface.sanitize_analysis(self.test_firmware.processed_analysis, self.test_firmware.get_uid())
self.assertIn('file_system_flag', sanitized_dict['stub_plugin'].keys())
self.assertFalse(sanitized_dict['stub_plugin']['file_system_flag'])
self.assertEqual(self.db_interface.sanitize_fs.list(), [], 'file stored in db but should not')
self.test_firmware.processed_analysis = long_dict
sanitized_dict = self.db_interface.sanitize_analysis(self.test_firmware.processed_analysis, self.test_firmware.get_uid())
self.assertIn('stub_plugin_result_{}'.format(self.test_firmware.get_uid()), self.db_interface.sanitize_fs.list(), 'sanitized file not stored')
self.assertNotIn('summary_result_{}'.format(self.test_firmware.get_uid()), self.db_interface.sanitize_fs.list(), 'summary is erroneously stored')
self.assertIn('file_system_flag', sanitized_dict['stub_plugin'].keys())
self.assertTrue(sanitized_dict['stub_plugin']['file_system_flag'])
self.assertEqual(type(sanitized_dict['stub_plugin']['summary']), list)
def test_retrieve_analysis(self):
self.db_interface.sanitize_fs.put(pickle.dumps('This is a test!'), filename='test_file_path')
sanitized_dict = {'stub_plugin': {'result': 'test_file_path', 'file_system_flag': True}}
sanitized_dict['inbound_result'] = {'result': 'inbound result', 'file_system_flag': False}
retrieved_dict = self.db_interface.retrieve_analysis(sanitized_dict)
self.assertNotIn('file_system_flag', retrieved_dict['stub_plugin'].keys())
self.assertIn('result', retrieved_dict['stub_plugin'].keys())
self.assertEqual(retrieved_dict['stub_plugin']['result'], 'This is a test!')
self.assertNotIn('file_system_flag', retrieved_dict['inbound_result'].keys())
self.assertEqual(retrieved_dict['inbound_result']['result'], 'inbound result')
def test_retrieve_analysis_filter(self):
self.db_interface.sanitize_fs.put(pickle.dumps('This is a test!'), filename='test_file_path')
sanitized_dict = {'selected_plugin': {'result': 'test_file_path', 'file_system_flag': True}}
sanitized_dict['other_plugin'] = {'result': 'test_file_path', 'file_system_flag': True}
retrieved_dict = self.db_interface.retrieve_analysis(sanitized_dict, analysis_filter=['selected_plugin'])
self.assertEqual(retrieved_dict['selected_plugin']['result'], 'This is a test!')
self.assertIn('file_system_flag', retrieved_dict['other_plugin'])
def test_get_objects_by_uid_list(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fo_list = self.db_interface.get_objects_by_uid_list([self.test_firmware.get_uid()])
self.assertIsInstance(fo_list[0], Firmware, 'firmware has wrong type')
self.assertEqual(fo_list[0].device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo_list = self.db_interface.get_objects_by_uid_list([test_file.get_uid()])
self.assertIsInstance(fo_list[0], FileObject, 'file object has wrong type')
def test_sanitize_extract_and_retrieve_binary(self):
test_data = {'dummy': {'test_key': 'test_value'}}
test_data['dummy'] = self.db_interface._extract_binaries(test_data, 'dummy', 'uid')
self.assertEqual(self.db_interface.sanitize_fs.list(), ['dummy_test_key_uid'], 'file not written')
self.assertEqual(test_data['dummy']['test_key'], 'dummy_test_key_uid', 'new file path not set')
test_data['dummy'] = self.db_interface._retrieve_binaries(test_data, 'dummy')
self.assertEqual(test_data['dummy']['test_key'], 'test_value', 'value not recoverd')
def test_get_firmware_number(self):
result = self.db_interface.get_firmware_number(query={})
self.assertEqual(result, 0)
self.db_interface_backend.add_firmware(self.test_firmware)
result = self.db_interface.get_firmware_number(query={})
self.assertEqual(result, 1)
result = self.db_interface.get_firmware_number(query={'_id': self.test_firmware.uid})
self.assertEqual(result, 1)
test_fw_2 = create_test_firmware(bin_path='container/test.7z')
self.db_interface_backend.add_firmware(test_fw_2)
result = self.db_interface.get_firmware_number(query='{}')
self.assertEqual(result, 2)
result = self.db_interface.get_firmware_number(query={'_id': self.test_firmware.uid})
self.assertEqual(result, 1)
def test_get_file_object_number(self):
result = self.db_interface.get_file_object_number(query={})
self.assertEqual(result, 0)
self.db_interface_backend.add_file_object(self.test_fo)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=False)
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query=json.dumps({'_id': self.test_fo.uid}))
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=True)
self.assertEqual(result, 0)
result = self.db_interface.get_file_object_number(query='{}', zero_on_empty_query=True)
self.assertEqual(result, 0)
test_fo_2 = create_test_file_object(bin_path='get_files_test/testfile2')
self.db_interface_backend.add_file_object(test_fo_2)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=False)
self.assertEqual(result, 2)
result = self.db_interface.get_file_object_number(query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(
config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'files_in_common': {
'all': [],
'collapse': False
}
},
'Software': {
'Compare Skipped': {
'all':
'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
parser = argparse.ArgumentParser(description="{} - {}".format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
parser.add_argument('-V', '--version', action='version', version="{} {}".format(PROGRAM_NAME, PROGRAM_VERSION))
parser.add_argument("-C", "--config_file", help="set path to config File", default="{}/main.cfg".format(get_config_dir()))
return parser.parse_args()
def _load_config(args):
config = configparser.ConfigParser()
config.read(args.config_file)
return config
def _setup_logging():
log_format = logging.Formatter(fmt="[%(asctime)s][%(module)s][%(levelname)s]: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger('')
logger.setLevel(logging.INFO)
console_logger = logging.StreamHandler()
console_logger.setFormatter(log_format)
logger.addHandler(console_logger)
if __name__ == '__main__':
args = _setup_argparser()
config = _load_config(args)
_setup_logging()
logging.info("Trying to start Mongo Server and initializing users...")
mongo_manger = MongoMgr(config=config, auth=False)
mongo_manger.init_users()
mongo_manger.shutdown()
class TestInterComTaskCommunication(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing(temp_dir=TMP_DIR)
self.config.set('ExpertSettings', 'communication_timeout', '1')
self.mongo_server = MongoMgr(config=self.config)
self.frontend = InterComFrontEndBinding(config=self.config)
self.backend = None
def tearDown(self):
for item in self.frontend.connections.keys():
self.frontend.client.drop_database(self.frontend.connections[item]['name'])
if self.backend:
self.backend.shutdown()
self.frontend.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def test_analysis_task(self):
self.backend = InterComBackEndAnalysisTask(config=self.config)
test_fw = create_test_firmware()
test_fw.file_path = None
self.frontend.add_analysis_task(test_fw)
task = self.backend.get_next_task()
self.assertEqual(task.get_uid(), test_fw.get_uid(), 'uid not correct')
self.assertIsNotNone(task.file_path, 'file_path not set')
self.assertTrue(os.path.exists(task.file_path), 'file does not exist')
def test_re_analyze_task(self):
self.backend = InterComBackEndReAnalyzeTask(config=self.config)
fs_organizer = FS_Organizer(config=self.config)
test_fw = create_test_firmware()
fs_organizer.store_file(test_fw)
original_file_path = test_fw.file_path
original_binary = test_fw.binary
test_fw.file_path = None
test_fw.binary = None
self.frontend.add_re_analyze_task(test_fw)
task = self.backend.get_next_task()
self.assertEqual(task.get_uid(), test_fw.get_uid(), 'uid not correct')
self.assertIsNotNone(task.file_path, 'file path not set')
self.assertEqual(task.file_path, original_file_path)
self.assertIsNotNone(task.binary, 'binary not set')
self.assertEqual(task.binary, original_binary, 'binary content not correct')
def test_compare_task(self):
self.backend = InterComBackEndCompareTask(config=self.config)
self.frontend.add_compare_task('valid_id', force=False)
result = self.backend.get_next_task()
self.assertEqual(result, ('valid_id', False))
def test_analysis_plugin_publication(self):
self.backend = InterComBackEndAnalysisPlugInsPublisher(config=self.config, analysis_service=AnalysisServiceMock())
plugins = self.frontend.get_available_analysis_plugins()
self.assertEqual(len(plugins), 1, 'Not all plug-ins found')
self.assertEqual(plugins, {'dummy': 'dummy description'}, 'content not correct')
def test_analysis_plugin_publication_not_available(self):
with self.assertRaises(Exception):
self.frontend.get_available_analysis_plugins()
@mock.patch('intercom.front_end_binding.generate_task_id')
@mock.patch('intercom.back_end_binding.BinaryService')
def test_raw_download_task(self, binaryServiceMock, generateTaskIdMock):
binaryServiceMock().get_binary_and_file_name.return_value = (b'test', 'test.txt')
generateTaskIdMock.return_value = 'valid_uid_0.0'
result = self.frontend.get_binary_and_filename('valid_uid')
self.assertIsNone(result, 'should be none because of timeout')
self.backend = InterComBackEndRawDownloadTask(config=self.config)
task = self.backend.get_next_task()
self.assertEqual(task, 'valid_uid', 'task not correct')
result = self.frontend.get_binary_and_filename('valid_uid_0.0')
self.assertEqual(result, (b'test', 'test.txt'), 'retrieved binary not correct')
@mock.patch('intercom.front_end_binding.generate_task_id')
@mock.patch('intercom.back_end_binding.BinaryService')
def test_tar_repack_task(self, binaryServiceMock, generateTaskIdMock):
binaryServiceMock().get_repacked_binary_and_file_name.return_value = (b'test', 'test.tar')
generateTaskIdMock.return_value = 'valid_uid_0.0'
result = self.frontend.get_repacked_binary_and_file_name('valid_uid')
self.assertIsNone(result, 'should be none because of timeout')
self.backend = InterComBackEndTarRepackTask(config=self.config)
task = self.backend.get_next_task()
self.assertEqual(task, 'valid_uid', 'task not correct')
result = self.frontend.get_repacked_binary_and_file_name('valid_uid_0.0')
self.assertEqual(result, (b'test', 'test.tar'), 'retrieved binary not correct')
class TestStorageDbInterfaceBackend(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def tearDown(self):
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface_backend.shutdown()
self.db_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def _get_all_firmware_uids(self):
uid_list = []
tmp = self.db_interface.firmwares.find()
for item in tmp:
uid_list.append(item['_id'])
return uid_list
def test_add_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertGreater(len(self._get_all_firmware_uids()), 0,
'No entry added to DB')
recoverd_firmware_entry = self.db_interface_backend.firmwares.find_one(
)
self.assertAlmostEqual(recoverd_firmware_entry['submission_date'],
time(),
msg='submission time not set correctly',
delta=5.0)
def test_add_and_get_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
result_backend = self.db_interface_backend.get_firmware(
self.test_firmware.get_uid())
self.assertIsNotNone(result_backend.binary,
'binary not set in backend result')
result_common = self.db_interface.get_firmware(
self.test_firmware.get_uid())
self.assertIsNone(result_common.binary, 'binary set in common result')
self.assertEqual(result_common.size, 787,
'file size not correct in common')
self.assertIsInstance(result_common.tags, dict,
'tag field type not correct')
def test_add_and_get_file_object(self):
self.db_interface_backend.add_file_object(self.test_fo)
result_backend = self.db_interface_backend.get_file_object(
self.test_fo.get_uid())
self.assertIsNotNone(result_backend.binary,
'binary not set in backend result')
result_common = self.db_interface.get_file_object(
self.test_fo.get_uid())
self.assertIsNone(result_common.binary, 'binary set in common result')
self.assertEqual(result_common.size, 62,
'file size not correct in common')
def test_update_firmware(self):
first_dict = {
'stub_plugin': {
'result': 0
},
'other_plugin': {
'field': 'day'
}
}
second_dict = {'stub_plugin': {'result': 1}}
self.test_firmware.processed_analysis = first_dict
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertEqual(
0,
self.db_interface.get_object(
self.test_firmware.get_uid()).processed_analysis['stub_plugin']
['result'])
self.test_firmware.processed_analysis = second_dict
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertEqual(
1,
self.db_interface.get_object(
self.test_firmware.get_uid()).processed_analysis['stub_plugin']
['result'])
self.assertIn(
'other_plugin',
self.db_interface.get_object(
self.test_firmware.get_uid()).processed_analysis.keys())
def test_update_file_object(self):
first_dict = {'other_plugin': {'result': 0}}
second_dict = {'stub_plugin': {'result': 1}}
self.test_fo.processed_analysis = first_dict
self.test_fo.files_included = {'file a', 'file b'}
self.db_interface_backend.add_file_object(self.test_fo)
self.test_fo.processed_analysis = second_dict
self.test_fo.files_included = {'file b', 'file c'}
self.db_interface_backend.add_file_object(self.test_fo)
received_object = self.db_interface.get_object(self.test_fo.get_uid())
self.assertEqual(
0, received_object.processed_analysis['other_plugin']['result'])
self.assertEqual(
1, received_object.processed_analysis['stub_plugin']['result'])
self.assertEqual(3, len(received_object.files_included))
def test_add_and_get_object_including_comment(self):
comment, author, date, uid = 'this is a test comment!', 'author', '1473431685', self.test_fo.get_uid(
)
self.test_fo.comments.append({
'time': str(date),
'author': author,
'comment': comment
})
self.db_interface_backend.add_file_object(self.test_fo)
retrieved_comment = self.db_interface.get_object(uid).comments[0]
self.assertEqual(author, retrieved_comment['author'])
self.assertEqual(comment, retrieved_comment['comment'])
self.assertEqual(date, retrieved_comment['time'])
username=config['data_storage']['db_admin_user'],
password=config['data_storage']['db_admin_pw'],
main_database=config['data_storage']['main_database'],
script_path=full_variety_path),
timeout=None)
logging.debug(output)
return return_code
if __name__ == '__main__':
args = _setup_argparser()
config = _load_config(args)
_setup_logging(args)
logging.info('Try to start Mongo Server...')
mongo_server = MongoMgr(config=config)
logging.info(
'updating data... this may take several hours depending on the size of your database'
)
start_time = time()
return_code = _create_variety_data(config)
process_time = time() - start_time
logging.info('generation time: {}'.format(time_format(process_time)))
if args.shutdown_db:
logging.info('Stopping Mongo Server...')
mongo_server.shutdown()
sys.exit(return_code)
class TestAcceptanceBase(unittest.TestCase):
class TestFW:
def __init__(self, uid, path, name):
self.uid = uid
self.path = path
self.name = name
self.file_name = os.path.basename(self.path)
def setUp(self):
self._set_config()
self.mongo_server = MongoMgr(config=self.config)
self.frontend = WebFrontEnd(config=self.config)
self.frontend.app.config['TESTING'] = True
self.test_client = self.frontend.app.test_client()
self.test_fw_a = self.TestFW(
'418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787',
'container/test.zip', 'test_fw_a')
self.test_fw_b = self.TestFW(
'd38970f8c5153d1041810d0908292bc8df21e7fd88aab211a8fb96c54afe6b01_319',
'container/test.7z', 'test_fw_b')
def tearDown(self):
clean_test_database(self.config, get_database_names(self.config))
self.mongo_server.shutdown()
gc.collect()
def _set_config(self):
self.config = load_config('main.cfg')
self.config.set('data_storage', 'main_database', TMP_DB_NAME)
self.config.set('data_storage', 'intercom_database_prefix',
TMP_DB_NAME)
self.config.set('data_storage', 'statistic_database', TMP_DB_NAME)
self.config.set('data_storage', 'firmware_file_storage_directory',
TMP_DIR.name)
self.config.set('Logging', 'mongoDbLogFile',
os.path.join(TMP_DIR.name, 'mongo.log'))
def _stop_backend(self):
self.intercom.shutdown()
self.compare_service.shutdown()
self.unpacking_service.shutdown()
self.analysis_service.shutdown()
def _start_backend(self):
self.analysis_service = AnalysisScheduler(config=self.config)
self.unpacking_service = UnpackingScheduler(
config=self.config, post_unpack=self.analysis_service.add_task)
self.compare_service = CompareScheduler(config=self.config)
self.intercom = InterComBackEndBinding(
config=self.config,
analysis_service=self.analysis_service,
compare_service=self.compare_service,
unpacking_service=self.unpacking_service)
def _setup_debugging_logging(self):
# for debugging purposes only
log_level = logging.DEBUG
log_format = logging.Formatter(
fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
create_dir_for_file(self.config['Logging']['logFile'])
file_log = logging.FileHandler(self.config['Logging']['logFile'])
file_log.setLevel(log_level)
file_log.setFormatter(log_format)
console_log = logging.StreamHandler()
console_log.setLevel(logging.DEBUG)
console_log.setFormatter(log_format)
logger.addHandler(file_log)
logger.addHandler(console_log)
class TestStorageDbInterfaceFrontend(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
def tearDown(self):
self.db_frontend_interface.shutdown()
self.db_backend_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_backend_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def test_get_meta_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
list_of_firmwares = self.db_frontend_interface.get_meta_list()
test_output = list_of_firmwares.pop()
self.assertEqual(test_output[1],
'test_vendor test_router - 0.1 (Router)',
'Firmware not successfully received')
def test_get_hid_firmware(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_hid(
self.test_firmware.get_uid())
self.assertEqual(result, 'test_vendor test_router - 0.1 (Router)',
'fw hid not correct')
def test_get_hid_fo(self):
test_fo = create_test_file_object(bin_path='get_files_test/testfile2')
test_fo.virtual_file_path = {
'a': ['|a|/test_file'],
'b': ['|b|/get_files_test/testfile2']
}
self.db_backend_interface.add_file_object(test_fo)
result = self.db_frontend_interface.get_hid(test_fo.get_uid(),
root_uid='b')
self.assertEqual(result, '/get_files_test/testfile2',
'fo hid not correct')
result = self.db_frontend_interface.get_hid(test_fo.get_uid())
self.assertIsInstance(result, str, 'result is not a string')
self.assertEqual(result[0], '/',
'first character not correct if no root_uid set')
result = self.db_frontend_interface.get_hid(test_fo.get_uid(),
root_uid='c')
self.assertEqual(
result[0], '/',
'first character not correct if invalid root_uid set')
def test_get_file_name(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_file_name(
self.test_firmware.get_uid())
self.assertEqual(result, 'test.zip', 'name not correct')
def test_get_hid_invalid_uid(self):
result = self.db_frontend_interface.get_hid('foo')
self.assertEqual(result, '',
'invalid uid should result in empty string')
def test_get_firmware_attribute_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertEqual(self.db_frontend_interface.get_device_class_list(),
['Router'])
self.assertEqual(self.db_frontend_interface.get_vendor_list(),
['test_vendor'])
self.assertEqual(
self.db_frontend_interface.get_firmware_attribute_list(
'device_name', {
'vendor': 'test_vendor',
'device_class': 'Router'
}), ['test_router'])
self.assertEqual(
self.db_frontend_interface.get_firmware_attribute_list('version'),
['0.1'])
self.assertEqual(self.db_frontend_interface.get_device_name_dict(),
{'Router': {
'test_vendor': ['test_router']
}})
def test_get_data_for_nice_list(self):
uid_list = [self.test_firmware.get_uid()]
self.db_backend_interface.add_firmware(self.test_firmware)
nice_list_data = self.db_frontend_interface.get_data_for_nice_list(
uid_list, uid_list[0])
self.assertEquals(
sorted([
'size', 'virtual_file_paths', 'uid', 'mime-type',
'files_included'
]), sorted(nice_list_data[0].keys()))
self.assertEqual(nice_list_data[0]['uid'],
self.test_firmware.get_uid())
def test_generic_search(self):
self.db_backend_interface.add_firmware(self.test_firmware)
# str input
result = self.db_frontend_interface.generic_search(
'{"file_name": "test.zip"}')
self.assertEqual(result, [self.test_firmware.get_uid()],
'Firmware not successfully received')
# dict input
result = self.db_frontend_interface.generic_search(
{'file_name': 'test.zip'})
self.assertEqual(result, [self.test_firmware.get_uid()],
'Firmware not successfully received')
def test_all_uids_found_in_database(self):
self.db_backend_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
uid_list = [self.test_firmware.get_uid()]
self.assertFalse(
self.db_frontend_interface.all_uids_found_in_database(uid_list))
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertTrue(
self.db_frontend_interface.all_uids_found_in_database(
[self.test_firmware.get_uid()]))
def test_get_number_of_firmwares_in_db(self):
self.assertEqual(
self.db_frontend_interface.get_number_of_firmwares_in_db(), 0)
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertEqual(
self.db_frontend_interface.get_number_of_firmwares_in_db(), 1)
def test_get_x_last_added_firmwares(self):
self.assertEqual(self.db_frontend_interface.get_last_added_firmwares(),
[], 'empty db should result in empty list')
test_fw_one = create_test_firmware(device_name='fw_one')
self.db_backend_interface.add_firmware(test_fw_one)
test_fw_two = create_test_firmware(device_name='fw_two',
bin_path='container/test.7z')
self.db_backend_interface.add_firmware(test_fw_two)
test_fw_three = create_test_firmware(device_name='fw_three',
bin_path='container/test.cab')
self.db_backend_interface.add_firmware(test_fw_three)
result = self.db_frontend_interface.get_last_added_firmwares(limit_x=2)
self.assertEqual(len(result), 2, 'Number of results should be 2')
self.assertEqual(result[0]['device_name'], 'fw_three',
'last firmware is not first entry')
self.assertEqual(result[1]['device_name'], 'fw_two',
'second last firmware is not the second entry')
def test_generate_file_tree_node(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
child_fo.processed_analysis['file_type'] = {'mime': 'sometype'}
uid = parent_fw.get_uid()
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
parent_fw.files_included = {child_fo.get_uid()}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
for node in self.db_frontend_interface.generate_file_tree_node(
uid, uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, parent_fw.file_name)
self.assertTrue(node.has_children)
for node in self.db_frontend_interface.generate_file_tree_node(
child_fo.get_uid(), uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, 'folder')
self.assertTrue(node.has_children)
virtual_grand_child = node.get_list_of_child_nodes()[0]
self.assertEqual(virtual_grand_child.type, 'sometype')
self.assertFalse(virtual_grand_child.has_children)
self.assertEqual(virtual_grand_child.name, child_fo.file_name)
def test_get_number_of_total_matches(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
uid = parent_fw.get_uid()
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(
uid, child_fo.get_uid())
self.assertEqual(
self.db_frontend_interface.get_number_of_total_matches(
query, only_parent_firmwares=False), 2)
self.assertEqual(
self.db_frontend_interface.get_number_of_total_matches(
query, only_parent_firmwares=True), 1)
def test_get_other_versions_of_firmware(self):
parent_fw1 = create_test_firmware(version='1')
self.db_backend_interface.add_object(parent_fw1)
parent_fw2 = create_test_firmware(version='2',
bin_path='container/test.7z')
self.db_backend_interface.add_object(parent_fw2)
parent_fw3 = create_test_firmware(version='3',
bin_path='container/test.cab')
self.db_backend_interface.add_object(parent_fw3)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(
parent_fw1)
self.assertEqual(len(other_versions), 2,
'wrong number of other versions')
self.assertIn({
'_id': parent_fw2.get_uid(),
'version': '2'
}, other_versions)
self.assertIn({
'_id': parent_fw3.get_uid(),
'version': '3'
}, other_versions)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(
parent_fw2)
self.assertIn({
'_id': parent_fw3.get_uid(),
'version': '3'
}, other_versions)
class TestStatistic(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.updater = StatisticUpdater(config=self.config)
self.frontend_db_interface = StatisticDbViewer(config=self.config)
def tearDown(self):
self.updater.shutdown()
self.frontend_db_interface.shutdown()
clean_test_database(self.config, get_database_names(self.config))
self.mongo_server.shutdown()
gc.collect()
def test_update_and_get_statistic(self):
self.updater.db.update_statistic('test', {'test1': 1})
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['test1'], 1, 'result not correct')
self.updater.db.update_statistic('test', {'test1': 2})
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['test1'], 2, 'result not correct')
def test_get_general_stats(self):
result = self.updater.get_general_stats()
self.assertEqual(result['number_of_firmwares'], 0,
'number of firmwares not correct')
self.assertEqual(result['number_of_unique_files'], 0,
'number of files not correct')
self.updater.db.firmwares.insert_one({'test': 1})
self.updater.db.file_objects.insert_one({'test': 1})
result = self.updater.get_general_stats()
self.assertEqual(result['number_of_firmwares'], 1,
'number of firmwares not correct')
self.assertEqual(result['number_of_unique_files'], 1,
'number of files not correct')
def test_convert_dict_list_to_list(self):
test_list = [{
'count': 1,
'_id': 'A'
}, {
'count': 2,
'_id': 'B'
}, {
'count': 3,
'_id': None
}]
result = self.updater._convert_dict_list_to_list(test_list)
self.assertIsInstance(result, list, 'result is not a list')
self.assertIn(['A', 1], result)
self.assertIn(['B', 2], result)
self.assertIn(['not available', 3], result)
self.assertEqual(len(result), 3, 'too many keys in the result')
def test_filter_sanitized_entries(self):
test_list = [
['valid', 1],
[
'sanitized_81abfc7a79c8c1ed85f6b9fc2c5d9a3edc4456c4aecb9f95b4d7a2bf9bf652da_1',
1
]
]
result = self.updater._filter_sanitzized_objects(test_list)
self.assertEqual(result, [['valid', 1]])
def test_find_most_frequent_architecture(self):
test_list = [
'MIPS, 32-bit, big endian (M)', 'MIPS (M)',
'MIPS, 32-bit, big endian (M)', 'MIPS, 32-bit, big endian (M)'
]
result = self.updater._find_most_frequent_architecture(test_list)
expected_result = 'MIPS, 32-bit, big endian (M)'
self.assertEqual(result, expected_result)
test_list = ['A', 'B', 'B', 'B', 'C', 'C']
result = self.updater._find_most_frequent_architecture(test_list)
expected_result = 'B'
self.assertEqual(result, expected_result)
def test_count_occurrences(self):
test_list = ['A', 'B', 'B', 'C', 'C', 'C']
result = set(self.updater._count_occurrences(test_list))
expected_result = {('A', 1), ('C', 3), ('B', 2)}
self.assertEqual(result, expected_result)
def test_shorten_architecture_string(self):
tests_string = 'MIPS, 64-bit, little endian (M)'
result = self.updater._shorten_architecture_string(tests_string)
self.assertEqual(result, 'MIPS, 64-bit')
tests_string = 'MIPS (M)'
result = self.updater._shorten_architecture_string(tests_string)
self.assertEqual(result, 'MIPS')
