def test_rule_expressions(self): """ Test generic rule on cuckoo report. """ config = '''[expressions] expression.1 : /DDE/ in cuckooreport.signature_descriptions -> bad ''' report = { "signatures": [{ "description": "Malicious document featuring Office DDE has been identified" }] } cuckooreport = CuckooReport(report) factory = SampleFactory(cuckoo=None, base_dir=None, job_hash_regex=None, keep_mail_data=False, processing_info_dir=None) sample = factory.make_sample('') sample.register_cuckoo_report(cuckooreport) rule = ExpressionRule(CreatingConfigParser(config)) result = rule.evaluate(sample) self.assertEqual(result.result, Result.bad)
def test_rule_ignore_mail_signatures(self): """ Test rule to ignore cryptographic mail signatures. """ config = '''[expressions] expression.4 : sample.meta_info_name_declared == 'smime.p7s' and sample.meta_info_type_declared in { 'application/pkcs7-signature', 'application/x-pkcs7-signature', 'application/pkcs7-mime', 'application/x-pkcs7-mime' } -> ignore expression.3 : sample.meta_info_name_declared == 'signature.asc' and sample.meta_info_type_declared in { 'application/pgp-signature' } -> ignore''' part = { "full_name": "p001", "name_declared": "smime.p7s", "type_declared": "application/pkcs7-signature" } factory = SampleFactory(cuckoo=None, base_dir=None, job_hash_regex=None, keep_mail_data=False, processing_info_dir=None) # test smime signatures sample = factory.make_sample('', metainfo=part) rule = ExpressionRule(CreatingConfigParser(config)) result = rule.evaluate(sample) self.assertEqual(result.result, Result.ignored) sample.meta_info_name_declared = "file" rule = ExpressionRule(CreatingConfigParser(config)) result = rule.evaluate(sample) self.assertEqual(result.result, Result.unknown) # test gpg signatures sample.meta_info_name_declared = "signature.asc" rule = ExpressionRule(CreatingConfigParser(config)) result = rule.evaluate(sample) self.assertEqual(result.result, Result.unknown) sample.meta_info_type_declared = "application/pgp-signature" rule = ExpressionRule(CreatingConfigParser(config)) result = rule.evaluate(sample) self.assertEqual(result.result, Result.ignored)
def setUpClass(cls): cls.test_db = os.path.abspath('./test.db') cls.conf = PeekabooDummyConfig() cls.db_con = PeekabooDatabase('sqlite:///' + cls.test_db) cls.factory = SampleFactory(cuckoo = None, db_con = cls.db_con, connection_map = None, base_dir = cls.conf.sample_base_dir, job_hash_regex = cls.conf.job_hash_regex, keep_mail_data = False) cls.sample = cls.factory.make_sample(os.path.realpath(__file__))
def setUpClass(cls): cls.test_db = os.path.abspath('./test.db') cls.conf = PeekabooDummyConfig() cls.db_con = PeekabooDatabase('sqlite:///' + cls.test_db) cls.factory = SampleFactory(cuckoo = None, db_con = cls.db_con, connection_map = None, base_dir = cls.conf.sample_base_dir, job_hash_regex = cls.conf.job_hash_regex, keep_mail_data = False) cls.sample = cls.factory.make_sample(os.path.realpath(__file__)) result = RuleResult('Unittest', Result.unknown, 'This is just a test case.', further_analysis=True) cls.sample.add_rule_result(result) cls.sample.determine_result()
async def async_main(): """ Runs the Peekaboo daemon. """ arg_parser = ArgumentParser( description='Peekaboo Extended Email Attachment Behavior Observation Owl' ) arg_parser.add_argument( '-c', '--config', action='store', help='The configuration file for Peekaboo.' ) arg_parser.add_argument( '-d', '--debug', action='store_true', help="Run Peekaboo in debug mode regardless of what's specified in the configuration." ) arg_parser.add_argument( '-D', '--daemon', action='store_true', help='Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).' ) args = arg_parser.parse_args() print('Starting Peekaboo %s.' % __version__) if not args.daemon: print(PEEKABOO_OWL) # Check if CLI arguments override the configuration log_level = None if args.debug: log_level = logging.DEBUG try: config = PeekabooConfig(config_file=args.config, log_level=log_level) logger.debug(config) except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # find localisation in our package directory locale_domain = 'peekaboo' locale_dir = os.path.join(os.path.dirname(__file__), 'locale') languages = None if config.report_locale: logger.debug('Looking for translations for preconfigured locale "%s"', config.report_locale) languages = [config.report_locale] if not gettext.find(locale_domain, locale_dir, languages): logger.warning('Translation file not found - falling back to ' 'system configuration.') languages = None logger.debug('Installing report message translations') translation = gettext.translation(locale_domain, locale_dir, languages, fallback=True) translation.install() # establish a connection to the database try: db_con = PeekabooDatabase( db_url=config.db_url, instance_id=config.cluster_instance_id, stale_in_flight_threshold=config.cluster_stale_in_flight_threshold, log_level=config.db_log_level) await db_con.start() except PeekabooDatabaseError as error: logging.critical(error) sys.exit(1) except SQLAlchemyError as dberr: logger.critical('Failed to establish a connection to the database ' 'at %s: %s', config.db_url, dberr) sys.exit(1) # initialize the daemon infrastructure such as PID file and dropping # privileges, automatically cleans up after itself when going out of scope daemon_infrastructure = PeekabooDaemonInfrastructure( config.pid_file, config.user, config.group) daemon_infrastructure.init() # clear all our in flight samples and all instances' stale in flight # samples await db_con.clear_in_flight_samples() await db_con.clear_stale_in_flight_samples() # a cluster duplicate interval of 0 disables the handler thread which is # what we want if we don't have an instance_id and therefore are alone cldup_check_interval = 0 if config.cluster_instance_id > 0: cldup_check_interval = config.cluster_duplicate_check_interval if cldup_check_interval < 5: cldup_check_interval = 5 logger.warning("Raising excessively low cluster duplicate check " "interval to %d seconds.", cldup_check_interval) loop = asyncio.get_running_loop() sig_handler = SignalHandler(loop) # separate threadpool for CPU- and I/O-bound blocking tasks (hashlib, # oletools, magic, requests, processing info dumping). This effectively # gives each of our asyncio Worker tasks an OS thread to execute blocking # operations on. The Queue might use them as well for stuff like # calculating samples' sha256sums, speeding up sample reception and # submission from the server. So maybe we should have some more threads # here... threadpool = concurrent.futures.ThreadPoolExecutor( config.worker_count, 'ThreadPool-') # collect a list of awaitables from started subsystems from which to gather # unexpected error conditions such as exceptions awaitables = [] # read in the analyzer and ruleset configuration and start the job queue try: ruleset_config = PeekabooConfigParser(config.ruleset_config) analyzer_config = PeekabooAnalyzerConfig(config.analyzer_config) job_queue = JobQueue( worker_count=config.worker_count, ruleset_config=ruleset_config, db_con=db_con, analyzer_config=analyzer_config, cluster_duplicate_check_interval=cldup_check_interval, threadpool=threadpool) sig_handler.register_listener(job_queue) awaitables.extend(await job_queue.start()) except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # Factory producing almost identical samples providing them with global # config values and references to other objects they need, such as database # connection and connection map. sample_factory = SampleFactory( config.processing_info_dir, threadpool) try: server = PeekabooServer( host=config.host, port=config.port, job_queue=job_queue, sample_factory=sample_factory, request_queue_size=100, db_con=db_con) sig_handler.register_listener(server) # the server runs completely inside the event loop and does not expose # any awaitable to extract exceptions from. await server.start() except Exception as error: logger.critical('Failed to start Peekaboo Server: %s', error) job_queue.shut_down() await job_queue.close_down() sys.exit(1) # abort startup if shutdown was requested meanwhile if sig_handler.shutdown_requested: sys.exit(0) SystemdNotifier().notify("READY=1") try: await asyncio.gather(*awaitables) # CancelledError is derived from BaseException, not Exception except asyncio.exceptions.CancelledError as error: # cancellation is expected in the case of shutdown via signal handler pass except Exception: logger.error("Shutting down due to unexpected exception") # trigger shutdowns of other components if not already ongoing triggered # by the signal handler if not sig_handler.shutdown_requested: server.shut_down() job_queue.shut_down() # close down components after they've shut down await server.close_down() await job_queue.close_down() # do a final cleanup pass through the database try: await db_con.clear_in_flight_samples() await db_con.clear_stale_in_flight_samples() except PeekabooDatabaseError as dberr: logger.error(dberr) sys.exit(0)
def run(): """ Runs the Peekaboo daemon. """ arg_parser = ArgumentParser( description= 'Peekaboo Extended Email Attachment Behavior Observation Owl') arg_parser.add_argument('-c', '--config', action='store', help='The configuration file for Peekaboo.') arg_parser.add_argument( '-d', '--debug', action='store_true', help= "Run Peekaboo in debug mode regardless of what's specified in the configuration." ) arg_parser.add_argument( '-D', '--daemon', action='store_true', help= 'Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).' ) args = arg_parser.parse_args() print('Starting Peekaboo %s.' % __version__) if not args.daemon: print(PEEKABOO_OWL) # Check if CLI arguments override the configuration log_level = None if args.debug: log_level = logging.DEBUG try: config = PeekabooConfig(config_file=args.config, log_level=log_level) logger.debug(config) except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # find localisation in our package directory locale_domain = 'peekaboo' locale_dir = os.path.join(os.path.dirname(__file__), 'locale') languages = None if config.report_locale: logger.debug('Looking for translations for preconfigured locale "%s"', config.report_locale) languages = [config.report_locale] if not gettext.find(locale_domain, locale_dir, languages): logger.warning('Translation file not found - falling back to ' 'system configuration.') languages = None logger.debug('Installing report message translations') translation = gettext.translation(locale_domain, locale_dir, languages, fallback=True) # python2's gettext needs to be told explicitly to return unicode strings loc_kwargs = {} if sys.version_info[0] < 3: loc_kwargs = {'unicode': True} translation.install(loc_kwargs) # establish a connection to the database try: db_con = PeekabooDatabase( db_url=config.db_url, instance_id=config.cluster_instance_id, stale_in_flight_threshold=config.cluster_stale_in_flight_threshold, log_level=config.db_log_level) except PeekabooDatabaseError as error: logging.critical(error) sys.exit(1) except SQLAlchemyError as dberr: logger.critical( 'Failed to establish a connection to the database ' 'at %s: %s', config.db_url, dberr) sys.exit(1) # Import debug module if we are in debug mode debugger = None if config.use_debug_module: from peekaboo.debug import PeekabooDebugger debugger = PeekabooDebugger() debugger.start() # initialize the daemon infrastructure such as PID file and dropping # privileges, automatically cleans up after itself when going out of scope daemon_infrastructure = PeekabooDaemonInfrastructure( config.pid_file, config.sock_file, config.user, config.group) daemon_infrastructure.init() systemd = SystemdNotifier() # clear all our in flight samples and all instances' stale in flight # samples db_con.clear_in_flight_samples() db_con.clear_stale_in_flight_samples() # a cluster duplicate interval of 0 disables the handler thread which is # what we want if we don't have an instance_id and therefore are alone cldup_check_interval = 0 if config.cluster_instance_id > 0: cldup_check_interval = config.cluster_duplicate_check_interval if cldup_check_interval < 5: cldup_check_interval = 5 logger.warning( "Raising excessively low cluster duplicate check " "interval to %d seconds.", cldup_check_interval) # workers of the job queue need the ruleset configuration to create the # ruleset engine with it try: ruleset_config = PeekabooConfigParser(config.ruleset_config) except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # verify the ruleset configuration by spawning a ruleset engine and having # it verify it try: engine = RulesetEngine(ruleset_config, db_con) except (KeyError, ValueError, PeekabooConfigException) as error: logging.critical('Ruleset configuration error: %s', error) sys.exit(1) except PeekabooRulesetConfigError as error: logging.critical(error) sys.exit(1) job_queue = JobQueue(worker_count=config.worker_count, ruleset_config=ruleset_config, db_con=db_con, cluster_duplicate_check_interval=cldup_check_interval) if config.cuckoo_mode == "embed": cuckoo = CuckooEmbed(job_queue, config.cuckoo_exec, config.cuckoo_submit, config.cuckoo_storage, config.interpreter) # otherwise it's the new API method and default else: cuckoo = CuckooApi(job_queue, config.cuckoo_url, config.cuckoo_api_token, config.cuckoo_poll_interval) sig_handler = SignalHandler() sig_handler.register_listener(cuckoo) # Factory producing almost identical samples providing them with global # config values and references to other objects they need, such as cuckoo, # database connection and connection map. sample_factory = SampleFactory(cuckoo, config.sample_base_dir, config.job_hash_regex, config.keep_mail_data, config.processing_info_dir) # We only want to accept 2 * worker_count connections. try: server = PeekabooServer(sock_file=config.sock_file, job_queue=job_queue, sample_factory=sample_factory, request_queue_size=config.worker_count * 2) except Exception as error: logger.critical('Failed to start Peekaboo Server: %s', error) job_queue.shut_down() if debugger is not None: debugger.shut_down() sys.exit(1) exit_code = 1 try: systemd.notify("READY=1") # If this dies Peekaboo dies, since this is the main thread. (legacy) exit_code = cuckoo.do() except Exception as error: logger.critical('Main thread aborted: %s', error) finally: server.shutdown() job_queue.shut_down() try: db_con.clear_in_flight_samples() db_con.clear_stale_in_flight_samples() except PeekabooDatabaseError as dberr: logger.error(dberr) if debugger is not None: debugger.shut_down() sys.exit(exit_code)
def test_rule_office_ole(self): """ Test rule office_ole. """ config = '''[office_macro_with_suspicious_keyword] keyword.1 : AutoOpen keyword.2 : AutoClose keyword.3 : suSPi.ious''' rule = OfficeMacroWithSuspiciousKeyword(CreatingConfigParser(config)) # sample factory to create samples from real files factory1 = SampleFactory(cuckoo=None, base_dir=None, job_hash_regex=None, keep_mail_data=False, processing_info_dir=None) # sampe factory to create samples with defined content factory2 = CreatingSampleFactory(cuckoo=None, base_dir=None, job_hash_regex=None, keep_mail_data=False, processing_info_dir=None) tests_data_dir = os.path.dirname( os.path.abspath(__file__)) + "/test-data" combinations = [ # no office document file extension [Result.unknown, factory2.make_sample('test.nodoc', 'test')], # test with empty file [ Result.unknown, factory1.make_sample(tests_data_dir + '/office/empty.doc') ], # office document with 'suspicious' in macro code [ Result.bad, factory1.make_sample(tests_data_dir + '/office/suspiciousMacro.doc') ], # test with blank word doc [ Result.unknown, factory1.make_sample(tests_data_dir + '/office/blank.doc') ], # test with legitimate macro [ Result.unknown, factory1.make_sample(tests_data_dir + '/office/legitmacro.xls') ] ] for expected, sample in combinations: result = rule.evaluate(sample) self.assertEqual(result.result, expected) # test if macro present rule = OfficeMacroRule(CreatingConfigParser(config)) combinations = [ # no office document file extension [Result.unknown, factory2.make_sample('test.nodoc', 'test')], # test with empty file [ Result.unknown, factory1.make_sample(tests_data_dir + '/office/empty.doc') ], # office document with 'suspicious' in macro code [ Result.bad, factory1.make_sample(tests_data_dir + '/office/suspiciousMacro.doc') ], # test with blank word doc [ Result.unknown, factory1.make_sample(tests_data_dir + '/office/blank.doc') ], # test with legitimate macro [ Result.bad, factory1.make_sample(tests_data_dir + '/office/legitmacro.xls') ] ] for expected, sample in combinations: result = rule.evaluate(sample) self.assertEqual(result.result, expected)
def run(): """ Runs the Peekaboo daemon. """ arg_parser = ArgumentParser( description='Peekaboo Extended Email Attachment Behavior Observation Owl' ) arg_parser.add_argument( '-c', '--config', action='store', help='The configuration file for Peekaboo.' ) arg_parser.add_argument( '-d', '--debug', action='store_true', help="Run Peekaboo in debug mode regardless of what's specified in the configuration." ) arg_parser.add_argument( '-D', '--daemon', action='store_true', help='Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).' ) args = arg_parser.parse_args() print('Starting Peekaboo %s.' % __version__) if not args.daemon: print(PEEKABOO_OWL) # Check if CLI arguments override the configuration log_level = None if args.debug: log_level = logging.DEBUG try: config = PeekabooConfig(config_file=args.config, log_level=log_level) logger.debug(config) except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # find localisation in our package directory locale_domain = 'peekaboo' locale_dir = os.path.join(os.path.dirname(__file__), 'locale') languages = None if config.report_locale: logger.debug('Looking for translations for preconfigured locale "%s"', config.report_locale) languages = [config.report_locale] if not gettext.find(locale_domain, locale_dir, languages): logger.warning('Translation file not found - falling back to ' 'system configuration.') languages = None logger.debug('Installing report message translations') translation = gettext.translation(locale_domain, locale_dir, languages, fallback=True) translation.install() # establish a connection to the database try: db_con = PeekabooDatabase( db_url=config.db_url, instance_id=config.cluster_instance_id, stale_in_flight_threshold=config.cluster_stale_in_flight_threshold, log_level=config.db_log_level) except PeekabooDatabaseError as error: logging.critical(error) sys.exit(1) except SQLAlchemyError as dberr: logger.critical('Failed to establish a connection to the database ' 'at %s: %s', config.db_url, dberr) sys.exit(1) # initialize the daemon infrastructure such as PID file and dropping # privileges, automatically cleans up after itself when going out of scope daemon_infrastructure = PeekabooDaemonInfrastructure( config.pid_file, config.sock_file, config.user, config.group) daemon_infrastructure.init() # clear all our in flight samples and all instances' stale in flight # samples db_con.clear_in_flight_samples() db_con.clear_stale_in_flight_samples() # a cluster duplicate interval of 0 disables the handler thread which is # what we want if we don't have an instance_id and therefore are alone cldup_check_interval = 0 if config.cluster_instance_id > 0: cldup_check_interval = config.cluster_duplicate_check_interval if cldup_check_interval < 5: cldup_check_interval = 5 logger.warning("Raising excessively low cluster duplicate check " "interval to %d seconds.", cldup_check_interval) sig_handler = SignalHandler() # read in the analyzer and ruleset configuration and start the job queue try: ruleset_config = PeekabooConfigParser(config.ruleset_config) analyzer_config = PeekabooAnalyzerConfig(config.analyzer_config) job_queue = JobQueue( worker_count=config.worker_count, ruleset_config=ruleset_config, db_con=db_con, analyzer_config=analyzer_config, cluster_duplicate_check_interval=cldup_check_interval) sig_handler.register_listener(job_queue) job_queue.start() except PeekabooConfigException as error: logging.critical(error) sys.exit(1) # Factory producing almost identical samples providing them with global # config values and references to other objects they need, such as database # connection and connection map. sample_factory = SampleFactory( config.sample_base_dir, config.job_hash_regex, config.keep_mail_data, config.processing_info_dir) # We only want to accept 2 * worker_count connections. try: server = PeekabooServer( sock_file=config.sock_file, job_queue=job_queue, sample_factory=sample_factory, request_queue_size=config.worker_count * 2, sock_group=config.sock_group, sock_mode=config.sock_mode) except Exception as error: logger.critical('Failed to start Peekaboo Server: %s', error) job_queue.shut_down() job_queue.close_down() sys.exit(1) # abort startup if shutdown was requested meanwhile if sig_handler.shutdown_requested: sys.exit(0) sig_handler.register_listener(server) SystemdNotifier().notify("READY=1") server.serve() # trigger shutdowns of other components (if not already ongoing triggered # by e.g. the signal handler), server will already be shut down at this # point signaled by the fact that serve() above returned job_queue.shut_down() # close down components after they've shut down job_queue.close_down() # do a final cleanup pass through the database try: db_con.clear_in_flight_samples() db_con.clear_stale_in_flight_samples() except PeekabooDatabaseError as dberr: logger.error(dberr) sys.exit(0)