class FactBase:
PROGRAM_NAME = 'FACT Base'
PROGRAM_DESCRIPTION = ''
COMPONENT = 'base'
def __init__(self):
self.run = True
set_signals(self.shutdown_listener)
self.args, self.config = program_setup(self.PROGRAM_NAME,
self.PROGRAM_DESCRIPTION)
self.work_load_stat = WorkLoadStatistic(config=self.config,
component=self.COMPONENT)
def shutdown_listener(self, signum, _):
logging.info(
f'Received signal {signum}. Shutting down {self.PROGRAM_NAME}...')
self.run = False
def shutdown(self):
logging.info(f'Shutting down components of {self.PROGRAM_NAME}')
self.work_load_stat.shutdown()
def main(self):
while self.run:
self.work_load_stat.update()
sleep(5)
if self.args.testing:
break
self.shutdown()
class TestWorkloadStatistic(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.workload_stat = WorkLoadStatistic(config=self.config,
component='test')
self.frontend_db_interface = StatisticDbViewer(config=self.config)
def tearDown(self):
self.frontend_db_interface.shutdown()
self.workload_stat.shutdown()
clean_test_database(self.config, get_database_names(self.config))
self.mongo_server.shutdown()
gc.collect()
def test_update_workload_statistic(self):
self.workload_stat.update()
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['name'], 'test', 'name not set')
self.assertAlmostEqual(time(),
result['last_update'],
msg='timestamp not valid',
delta=100)
self.assertIsInstance(result['platform'], dict,
'platfom is not a dict')
self.assertIsInstance(result['system'], dict, 'system is not a dict')
class TestRestStatistic(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.workload = WorkLoadStatistic(config=self.config)
sleep(1) # wait for systems to start
def tearDown(self):
self.workload.shutdown()
self._stop_backend()
super().tearDown()
def test_status(self):
self.workload.update(
unpacking_workload=self.unpacking_service.get_scheduled_workload(),
analysis_workload=self.analysis_service.get_scheduled_workload())
rv = self.test_client.get('/rest/status', follow_redirects=True)
assert rv.status_code == 200
assert all(key in rv.json for key in ['system_status', 'plugins'])
assert 'backend' in rv.json['system_status']
assert rv.json['system_status']['backend']['status'] == 'online'
assert rv.json['system_status']['backend']['analysis']['plugins'][
'file_type']['queue'] == 0
def __init__(self):
self.run = True
set_signals(self.shutdown_listener)
self.args, self.config = program_setup(self.PROGRAM_NAME,
self.PROGRAM_DESCRIPTION)
self.work_load_stat = WorkLoadStatistic(config=self.config,
component=self.COMPONENT)
def setUp(self):
super().setUp()
self._start_backend(post_analysis=self._analysis_callback)
self.updater = StatisticUpdater(config=self.config)
self.workload = WorkLoadStatistic(config=self.config,
component='backend')
time.sleep(2) # wait for systems to start
def __init__(self):
self.run = True
# Check whether the process was started by start_fact.py
parent = ' '.join(psutil.Process(os.getppid()).cmdline())
started_by_start_fact_py = 'start_fact.py' in parent or 'start_all_installed_fact_components' in parent
if started_by_start_fact_py:
signal.signal(signal.SIGUSR1, self.shutdown_listener)
signal.signal(signal.SIGINT, lambda *_: None)
os.setpgid(os.getpid(), os.getpid()) # reset pgid to self so that "complete_shutdown" doesn't run amok
else:
signal.signal(signal.SIGINT, self.shutdown_listener)
self.args, self.config = program_setup(self.PROGRAM_NAME, self.PROGRAM_DESCRIPTION, self.COMPONENT)
self.work_load_stat = WorkLoadStatistic(config=self.config, component=self.COMPONENT)
class FactBase:
PROGRAM_NAME = 'FACT Base'
PROGRAM_DESCRIPTION = ''
COMPONENT = 'base'
def __init__(self):
self.run = True
# Check whether the process was started by start_fact.py
parent = ' '.join(psutil.Process(os.getppid()).cmdline())
started_by_start_fact_py = 'start_fact.py' in parent or 'start_all_installed_fact_components' in parent
if started_by_start_fact_py:
signal.signal(signal.SIGUSR1, self.shutdown_listener)
signal.signal(signal.SIGINT, lambda *_: None)
os.setpgid(os.getpid(), os.getpid()) # reset pgid to self so that "complete_shutdown" doesn't run amok
else:
signal.signal(signal.SIGINT, self.shutdown_listener)
self.args, self.config = program_setup(self.PROGRAM_NAME, self.PROGRAM_DESCRIPTION, self.COMPONENT)
self.work_load_stat = WorkLoadStatistic(config=self.config, component=self.COMPONENT)
def shutdown_listener(self, signum, _):
logging.info(f'Received signal {signum}. Shutting down {self.PROGRAM_NAME}...')
self.run = False
def shutdown(self):
logging.info(f'Shutting down components of {self.PROGRAM_NAME}')
self.work_load_stat.shutdown()
def main(self):
logging.info(f'Successfully started {self.PROGRAM_NAME}')
while self.run:
self.work_load_stat.update()
sleep(5)
if self.args.testing:
break
self.shutdown()
if __name__ == '__main__':
if was_started_by_start_fact():
signal.signal(signal.SIGUSR1, shutdown)
signal.signal(signal.SIGINT, lambda *_: None)
os.setpgid(os.getpid(), os.getpid()) # reset pgid to self so that "complete_shutdown" doesn't run amok
else:
signal.signal(signal.SIGINT, shutdown)
args, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION)
analysis_service = AnalysisScheduler(config=config)
tagging_service = TaggingDaemon(analysis_scheduler=analysis_service)
unpacking_service = UnpackingScheduler(config=config, post_unpack=analysis_service.start_analysis_of_object, analysis_workload=analysis_service.get_scheduled_workload)
compare_service = CompareScheduler(config=config)
intercom = InterComBackEndBinding(config=config, analysis_service=analysis_service, compare_service=compare_service, unpacking_service=unpacking_service)
work_load_stat = WorkLoadStatistic(config=config)
run = True
while run:
work_load_stat.update(unpacking_workload=unpacking_service.get_scheduled_workload(), analysis_workload=analysis_service.get_scheduled_workload())
if any((unpacking_service.check_exceptions(), compare_service.check_exceptions(), analysis_service.check_exceptions())):
break
sleep(5)
if args.testing:
break
logging.info('shutdown components')
work_load_stat.shutdown()
intercom.shutdown()
compare_service.shutdown()
unpacking_service.shutdown()
class TestAcceptanceShowStatsAndSystemMonitor(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.updater = StatisticUpdater(config=self.config)
self.workload = WorkLoadStatistic(config=self.config)
time.sleep(10) # wait for systems to start
def tearDown(self):
self.updater.shutdown()
self._stop_backend()
super().tearDown()
def _upload_firmware_get(self):
rv = self.test_client.get('/upload')
self.assertIn(b'<h2>Upload Firmware</h2>', rv.data, "upload page not displayed correctly")
def _upload_firmware_put(self, path, device_name, uid):
testfile_path = os.path.join(get_test_data_dir(), path)
with open(testfile_path, "rb") as fp:
data = {
'file': fp,
'device_name': device_name,
'device_class': "test_class",
'firmware_version': "1.0",
'vendor': "test_vendor",
'release_date': "2009-01-01",
'tags': '',
'analysis_systems': []
}
rv = self.test_client.post('/upload', content_type='multipart/form-data', data=data, follow_redirects=True)
self.assertIn(b'Upload Successful', rv.data, "upload not successful")
self.assertIn(uid.encode(), rv.data, "uid not found on upload success page")
def _show_stats(self):
rv = self.test_client.get('/statistic')
self.assertIn(b'Firmware Container', rv.data)
self.assertIn(b'test_vendor', rv.data)
self.assertIn(b'Release Date Stats', rv.data)
def _show_stats_filtered(self):
rv = self.test_client.get('/statistic?vendor=test_vendor')
self.assertIn(b'Firmware Container', rv.data)
self.assertIn(b'test_vendor', rv.data)
self.assertIn(b'Release Date Stats', rv.data)
def _click_chart(self):
rv = self.test_client.get('/database/browse?query=%7b%22vendor%22%3A+%7b%22%24eq%22%3A+%22test_vendor%22%7d%7d')
self.assertIn(self.test_fw_a.uid.encode(), rv.data)
def _click_release_date_histogram(self):
rv = self.test_client.get('/database/browse?date="January 2009"')
self.assertIn(self.test_fw_a.uid.encode(), rv.data)
def _show_system_monitor(self):
rv = self.test_client.get('/system_health')
self.assertIn(b'backend workload', rv.data)
def test_show_stats_and_system_monitor(self):
self._upload_firmware_get()
for fw in [self.test_fw_a, self.test_fw_b]:
self._upload_firmware_put(fw.path, fw.name, fw.uid)
time.sleep(5)
self.workload.update(unpacking_workload=self.unpacking_service.get_scheduled_workload(), analysis_workload=self.analysis_service.get_scheduled_workload())
time.sleep(10)
self._show_system_monitor()
self.updater.update_all_stats()
self._show_stats()
self._show_stats_filtered()
self._click_chart()
self._click_release_date_histogram()
def setUp(self):
super().setUp()
self._start_backend()
self.updater = StatisticUpdater(config=self.config)
self.workload = WorkLoadStatistic(config=self.config)
time.sleep(10) # wait for systems to start
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.workload_stat = WorkLoadStatistic(config=self.config,
component='test')
self.frontend_db_interface = StatisticDbViewer(config=self.config)
class TestAcceptanceMisc(TestAcceptanceBase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.db_backend_service = BackEndDbInterface(config=cls.config)
cls.analysis_finished_event = Event()
cls.elements_finished_analyzing = Value('i', 0)
def setUp(self):
super().setUp()
self._start_backend(post_analysis=self._analysis_callback)
self.updater = StatisticUpdater(config=self.config)
self.workload = WorkLoadStatistic(config=self.config)
time.sleep(2) # wait for systems to start
def tearDown(self):
self.updater.shutdown()
self._stop_backend()
super().tearDown()
@classmethod
def tearDownClass(cls):
cls.db_backend_service.shutdown()
super().tearDownClass()
def _analysis_callback(self, fo):
self.db_backend_service.add_object(fo)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value > 7:
self.analysis_finished_event.set()
def _upload_firmware_get(self):
rv = self.test_client.get('/upload')
self.assertIn(b'<h2>Upload Firmware</h2>', rv.data,
"upload page not displayed correctly")
def _upload_firmware_put(self, path, device_name, uid):
testfile_path = os.path.join(get_test_data_dir(), path)
with open(testfile_path, "rb") as fp:
data = {
'file': fp,
'device_name': device_name,
'device_class': "test_class",
'firmware_version': "1.0",
'vendor': "test_vendor",
'release_date': "2009-01-01",
'tags': '',
'analysis_systems': []
}
rv = self.test_client.post('/upload',
content_type='multipart/form-data',
data=data,
follow_redirects=True)
self.assertIn(b'Upload Successful', rv.data, "upload not successful")
self.assertIn(uid.encode(), rv.data,
"uid not found on upload success page")
def _show_stats(self):
rv = self.test_client.get('/statistic')
self.assertIn(b'Firmware Container', rv.data)
self.assertIn(b'test_vendor', rv.data)
self.assertIn(b'Release Date Stats', rv.data)
def _show_stats_filtered(self):
rv = self.test_client.get('/statistic?vendor=test_vendor')
self.assertIn(b'Firmware Container', rv.data)
self.assertIn(b'test_vendor', rv.data)
self.assertIn(b'Release Date Stats', rv.data)
def _show_about(self):
rv = self.test_client.get('/about')
self.assertIn(b'License Information', rv.data)
def _show_system_monitor(self):
rv = self.test_client.get('/system_health')
self.assertIn(b'backend workload', rv.data)
def _click_chart(self):
rv = self.test_client.get(
'/database/browse?query=%7b%22vendor%22%3A+%7b%22%24eq%22%3A+%22test_vendor%22%7d%7d'
)
self.assertIn(self.test_fw_a.uid.encode(), rv.data)
def _click_release_date_histogram(self):
rv = self.test_client.get('/database/browse?date="January 2009"')
self.assertIn(self.test_fw_a.uid.encode(), rv.data)
def test_misc(self):
self._upload_firmware_get()
for fw in [self.test_fw_a, self.test_fw_b]:
self._upload_firmware_put(fw.path, fw.name, fw.uid)
self._show_about()
time.sleep(4)
self.workload.update(
unpacking_workload=self.unpacking_service.get_scheduled_workload(),
analysis_workload=self.analysis_service.get_scheduled_workload())
self.analysis_finished_event.wait(timeout=10)
self._show_system_monitor()
self.updater.update_all_stats()
self._show_stats()
self._show_stats_filtered()
self._click_chart()
self._click_release_date_histogram()
def shutdown(*_):
global run
logging.info('shutting down {}...'.format(PROGRAM_NAME))
run = False
if __name__ == '__main__':
if was_started_by_start_fact():
signal.signal(signal.SIGUSR1, shutdown)
signal.signal(signal.SIGINT, lambda *_: None)
else:
signal.signal(signal.SIGINT, shutdown)
args, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION)
mongo_server = MongoMgr(config=config)
work_load_stat = WorkLoadStatistic(config=config, component='database')
run = True
while run:
work_load_stat.update()
sleep(5)
if args.testing:
break
work_load_stat.shutdown()
mongo_server.shutdown()
sys.exit()
def start_uwsgi_server(config_path=None):
config_parameter = ' --pyargv {}'.format(
config_path) if config_path else ''
p = Popen('(cd {} && uwsgi --ini {}/uwsgi_config.ini{})'.format(
get_src_dir(), get_config_dir(), config_parameter),
shell=True)
return p
if __name__ == '__main__':
run = True
args = _setup_argparser(name=PROGRAM_NAME, description=PROGRAM_DESCRIPTION)
config = _load_config(args)
_setup_logging(config, args)
work_load_stat = WorkLoadStatistic(config=config, component='frontend')
with tempfile.NamedTemporaryFile() as fp:
fp.write(pickle.dumps(args))
fp.flush()
uwsgi_process = start_uwsgi_server(fp.name)
while run:
work_load_stat.update()
sleep(5)
if args.testing:
break
work_load_stat.shutdown()
_shutdown_uwsgi_server(uwsgi_process)
sys.exit()
def setUp(self):
super().setUp()
self._start_backend()
self.workload = WorkLoadStatistic(config=self.config)
sleep(1) # wait for systems to start