def test_rule_expressions(self):
""" Test generic rule on cuckoo report. """
config = '''[expressions]
expression.1 : /DDE/ in cuckooreport.signature_descriptions -> bad
'''
report = {
"signatures": [{
"description":
"Malicious document featuring Office DDE has been identified"
}]
}
cuckooreport = CuckooReport(report)
factory = SampleFactory(cuckoo=None,
base_dir=None,
job_hash_regex=None,
keep_mail_data=False,
processing_info_dir=None)
sample = factory.make_sample('')
sample.register_cuckoo_report(cuckooreport)
rule = ExpressionRule(CreatingConfigParser(config))
result = rule.evaluate(sample)
self.assertEqual(result.result, Result.bad)
def test_rule_ignore_mail_signatures(self):
""" Test rule to ignore cryptographic mail signatures. """
config = '''[expressions]
expression.4 : sample.meta_info_name_declared == 'smime.p7s'
and sample.meta_info_type_declared in {
'application/pkcs7-signature',
'application/x-pkcs7-signature',
'application/pkcs7-mime',
'application/x-pkcs7-mime'
} -> ignore
expression.3 : sample.meta_info_name_declared == 'signature.asc'
and sample.meta_info_type_declared in {
'application/pgp-signature'
} -> ignore'''
part = {
"full_name": "p001",
"name_declared": "smime.p7s",
"type_declared": "application/pkcs7-signature"
}
factory = SampleFactory(cuckoo=None,
base_dir=None,
job_hash_regex=None,
keep_mail_data=False,
processing_info_dir=None)
# test smime signatures
sample = factory.make_sample('', metainfo=part)
rule = ExpressionRule(CreatingConfigParser(config))
result = rule.evaluate(sample)
self.assertEqual(result.result, Result.ignored)
sample.meta_info_name_declared = "file"
rule = ExpressionRule(CreatingConfigParser(config))
result = rule.evaluate(sample)
self.assertEqual(result.result, Result.unknown)
# test gpg signatures
sample.meta_info_name_declared = "signature.asc"
rule = ExpressionRule(CreatingConfigParser(config))
result = rule.evaluate(sample)
self.assertEqual(result.result, Result.unknown)
sample.meta_info_type_declared = "application/pgp-signature"
rule = ExpressionRule(CreatingConfigParser(config))
result = rule.evaluate(sample)
self.assertEqual(result.result, Result.ignored)
def setUpClass(cls):
cls.test_db = os.path.abspath('./test.db')
cls.conf = PeekabooDummyConfig()
cls.db_con = PeekabooDatabase('sqlite:///' + cls.test_db)
cls.factory = SampleFactory(cuckoo = None,
db_con = cls.db_con,
connection_map = None,
base_dir = cls.conf.sample_base_dir,
job_hash_regex = cls.conf.job_hash_regex,
keep_mail_data = False)
cls.sample = cls.factory.make_sample(os.path.realpath(__file__))
def setUpClass(cls):
cls.test_db = os.path.abspath('./test.db')
cls.conf = PeekabooDummyConfig()
cls.db_con = PeekabooDatabase('sqlite:///' + cls.test_db)
cls.factory = SampleFactory(cuckoo = None,
db_con = cls.db_con,
connection_map = None,
base_dir = cls.conf.sample_base_dir,
job_hash_regex = cls.conf.job_hash_regex,
keep_mail_data = False)
cls.sample = cls.factory.make_sample(os.path.realpath(__file__))
result = RuleResult('Unittest',
Result.unknown,
'This is just a test case.',
further_analysis=True)
cls.sample.add_rule_result(result)
cls.sample.determine_result()
async def async_main():
""" Runs the Peekaboo daemon. """
arg_parser = ArgumentParser(
description='Peekaboo Extended Email Attachment Behavior Observation Owl'
)
arg_parser.add_argument(
'-c', '--config',
action='store',
help='The configuration file for Peekaboo.'
)
arg_parser.add_argument(
'-d', '--debug',
action='store_true',
help="Run Peekaboo in debug mode regardless of what's specified in the configuration."
)
arg_parser.add_argument(
'-D', '--daemon',
action='store_true',
help='Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).'
)
args = arg_parser.parse_args()
print('Starting Peekaboo %s.' % __version__)
if not args.daemon:
print(PEEKABOO_OWL)
# Check if CLI arguments override the configuration
log_level = None
if args.debug:
log_level = logging.DEBUG
try:
config = PeekabooConfig(config_file=args.config, log_level=log_level)
logger.debug(config)
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# find localisation in our package directory
locale_domain = 'peekaboo'
locale_dir = os.path.join(os.path.dirname(__file__), 'locale')
languages = None
if config.report_locale:
logger.debug('Looking for translations for preconfigured locale "%s"',
config.report_locale)
languages = [config.report_locale]
if not gettext.find(locale_domain, locale_dir, languages):
logger.warning('Translation file not found - falling back to '
'system configuration.')
languages = None
logger.debug('Installing report message translations')
translation = gettext.translation(locale_domain, locale_dir, languages,
fallback=True)
translation.install()
# establish a connection to the database
try:
db_con = PeekabooDatabase(
db_url=config.db_url, instance_id=config.cluster_instance_id,
stale_in_flight_threshold=config.cluster_stale_in_flight_threshold,
log_level=config.db_log_level)
await db_con.start()
except PeekabooDatabaseError as error:
logging.critical(error)
sys.exit(1)
except SQLAlchemyError as dberr:
logger.critical('Failed to establish a connection to the database '
'at %s: %s', config.db_url, dberr)
sys.exit(1)
# initialize the daemon infrastructure such as PID file and dropping
# privileges, automatically cleans up after itself when going out of scope
daemon_infrastructure = PeekabooDaemonInfrastructure(
config.pid_file, config.user, config.group)
daemon_infrastructure.init()
# clear all our in flight samples and all instances' stale in flight
# samples
await db_con.clear_in_flight_samples()
await db_con.clear_stale_in_flight_samples()
# a cluster duplicate interval of 0 disables the handler thread which is
# what we want if we don't have an instance_id and therefore are alone
cldup_check_interval = 0
if config.cluster_instance_id > 0:
cldup_check_interval = config.cluster_duplicate_check_interval
if cldup_check_interval < 5:
cldup_check_interval = 5
logger.warning("Raising excessively low cluster duplicate check "
"interval to %d seconds.",
cldup_check_interval)
loop = asyncio.get_running_loop()
sig_handler = SignalHandler(loop)
# separate threadpool for CPU- and I/O-bound blocking tasks (hashlib,
# oletools, magic, requests, processing info dumping). This effectively
# gives each of our asyncio Worker tasks an OS thread to execute blocking
# operations on. The Queue might use them as well for stuff like
# calculating samples' sha256sums, speeding up sample reception and
# submission from the server. So maybe we should have some more threads
# here...
threadpool = concurrent.futures.ThreadPoolExecutor(
config.worker_count, 'ThreadPool-')
# collect a list of awaitables from started subsystems from which to gather
# unexpected error conditions such as exceptions
awaitables = []
# read in the analyzer and ruleset configuration and start the job queue
try:
ruleset_config = PeekabooConfigParser(config.ruleset_config)
analyzer_config = PeekabooAnalyzerConfig(config.analyzer_config)
job_queue = JobQueue(
worker_count=config.worker_count, ruleset_config=ruleset_config,
db_con=db_con, analyzer_config=analyzer_config,
cluster_duplicate_check_interval=cldup_check_interval,
threadpool=threadpool)
sig_handler.register_listener(job_queue)
awaitables.extend(await job_queue.start())
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# Factory producing almost identical samples providing them with global
# config values and references to other objects they need, such as database
# connection and connection map.
sample_factory = SampleFactory(
config.processing_info_dir, threadpool)
try:
server = PeekabooServer(
host=config.host, port=config.port,
job_queue=job_queue,
sample_factory=sample_factory,
request_queue_size=100,
db_con=db_con)
sig_handler.register_listener(server)
# the server runs completely inside the event loop and does not expose
# any awaitable to extract exceptions from.
await server.start()
except Exception as error:
logger.critical('Failed to start Peekaboo Server: %s', error)
job_queue.shut_down()
await job_queue.close_down()
sys.exit(1)
# abort startup if shutdown was requested meanwhile
if sig_handler.shutdown_requested:
sys.exit(0)
SystemdNotifier().notify("READY=1")
try:
await asyncio.gather(*awaitables)
# CancelledError is derived from BaseException, not Exception
except asyncio.exceptions.CancelledError as error:
# cancellation is expected in the case of shutdown via signal handler
pass
except Exception:
logger.error("Shutting down due to unexpected exception")
# trigger shutdowns of other components if not already ongoing triggered
# by the signal handler
if not sig_handler.shutdown_requested:
server.shut_down()
job_queue.shut_down()
# close down components after they've shut down
await server.close_down()
await job_queue.close_down()
# do a final cleanup pass through the database
try:
await db_con.clear_in_flight_samples()
await db_con.clear_stale_in_flight_samples()
except PeekabooDatabaseError as dberr:
logger.error(dberr)
sys.exit(0)
def run():
""" Runs the Peekaboo daemon. """
arg_parser = ArgumentParser(
description=
'Peekaboo Extended Email Attachment Behavior Observation Owl')
arg_parser.add_argument('-c',
'--config',
action='store',
help='The configuration file for Peekaboo.')
arg_parser.add_argument(
'-d',
'--debug',
action='store_true',
help=
"Run Peekaboo in debug mode regardless of what's specified in the configuration."
)
arg_parser.add_argument(
'-D',
'--daemon',
action='store_true',
help=
'Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).'
)
args = arg_parser.parse_args()
print('Starting Peekaboo %s.' % __version__)
if not args.daemon:
print(PEEKABOO_OWL)
# Check if CLI arguments override the configuration
log_level = None
if args.debug:
log_level = logging.DEBUG
try:
config = PeekabooConfig(config_file=args.config, log_level=log_level)
logger.debug(config)
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# find localisation in our package directory
locale_domain = 'peekaboo'
locale_dir = os.path.join(os.path.dirname(__file__), 'locale')
languages = None
if config.report_locale:
logger.debug('Looking for translations for preconfigured locale "%s"',
config.report_locale)
languages = [config.report_locale]
if not gettext.find(locale_domain, locale_dir, languages):
logger.warning('Translation file not found - falling back to '
'system configuration.')
languages = None
logger.debug('Installing report message translations')
translation = gettext.translation(locale_domain,
locale_dir,
languages,
fallback=True)
# python2's gettext needs to be told explicitly to return unicode strings
loc_kwargs = {}
if sys.version_info[0] < 3:
loc_kwargs = {'unicode': True}
translation.install(loc_kwargs)
# establish a connection to the database
try:
db_con = PeekabooDatabase(
db_url=config.db_url,
instance_id=config.cluster_instance_id,
stale_in_flight_threshold=config.cluster_stale_in_flight_threshold,
log_level=config.db_log_level)
except PeekabooDatabaseError as error:
logging.critical(error)
sys.exit(1)
except SQLAlchemyError as dberr:
logger.critical(
'Failed to establish a connection to the database '
'at %s: %s', config.db_url, dberr)
sys.exit(1)
# Import debug module if we are in debug mode
debugger = None
if config.use_debug_module:
from peekaboo.debug import PeekabooDebugger
debugger = PeekabooDebugger()
debugger.start()
# initialize the daemon infrastructure such as PID file and dropping
# privileges, automatically cleans up after itself when going out of scope
daemon_infrastructure = PeekabooDaemonInfrastructure(
config.pid_file, config.sock_file, config.user, config.group)
daemon_infrastructure.init()
systemd = SystemdNotifier()
# clear all our in flight samples and all instances' stale in flight
# samples
db_con.clear_in_flight_samples()
db_con.clear_stale_in_flight_samples()
# a cluster duplicate interval of 0 disables the handler thread which is
# what we want if we don't have an instance_id and therefore are alone
cldup_check_interval = 0
if config.cluster_instance_id > 0:
cldup_check_interval = config.cluster_duplicate_check_interval
if cldup_check_interval < 5:
cldup_check_interval = 5
logger.warning(
"Raising excessively low cluster duplicate check "
"interval to %d seconds.", cldup_check_interval)
# workers of the job queue need the ruleset configuration to create the
# ruleset engine with it
try:
ruleset_config = PeekabooConfigParser(config.ruleset_config)
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# verify the ruleset configuration by spawning a ruleset engine and having
# it verify it
try:
engine = RulesetEngine(ruleset_config, db_con)
except (KeyError, ValueError, PeekabooConfigException) as error:
logging.critical('Ruleset configuration error: %s', error)
sys.exit(1)
except PeekabooRulesetConfigError as error:
logging.critical(error)
sys.exit(1)
job_queue = JobQueue(worker_count=config.worker_count,
ruleset_config=ruleset_config,
db_con=db_con,
cluster_duplicate_check_interval=cldup_check_interval)
if config.cuckoo_mode == "embed":
cuckoo = CuckooEmbed(job_queue, config.cuckoo_exec,
config.cuckoo_submit, config.cuckoo_storage,
config.interpreter)
# otherwise it's the new API method and default
else:
cuckoo = CuckooApi(job_queue, config.cuckoo_url,
config.cuckoo_api_token,
config.cuckoo_poll_interval)
sig_handler = SignalHandler()
sig_handler.register_listener(cuckoo)
# Factory producing almost identical samples providing them with global
# config values and references to other objects they need, such as cuckoo,
# database connection and connection map.
sample_factory = SampleFactory(cuckoo, config.sample_base_dir,
config.job_hash_regex,
config.keep_mail_data,
config.processing_info_dir)
# We only want to accept 2 * worker_count connections.
try:
server = PeekabooServer(sock_file=config.sock_file,
job_queue=job_queue,
sample_factory=sample_factory,
request_queue_size=config.worker_count * 2)
except Exception as error:
logger.critical('Failed to start Peekaboo Server: %s', error)
job_queue.shut_down()
if debugger is not None:
debugger.shut_down()
sys.exit(1)
exit_code = 1
try:
systemd.notify("READY=1")
# If this dies Peekaboo dies, since this is the main thread. (legacy)
exit_code = cuckoo.do()
except Exception as error:
logger.critical('Main thread aborted: %s', error)
finally:
server.shutdown()
job_queue.shut_down()
try:
db_con.clear_in_flight_samples()
db_con.clear_stale_in_flight_samples()
except PeekabooDatabaseError as dberr:
logger.error(dberr)
if debugger is not None:
debugger.shut_down()
sys.exit(exit_code)
def test_rule_office_ole(self):
""" Test rule office_ole. """
config = '''[office_macro_with_suspicious_keyword]
keyword.1 : AutoOpen
keyword.2 : AutoClose
keyword.3 : suSPi.ious'''
rule = OfficeMacroWithSuspiciousKeyword(CreatingConfigParser(config))
# sample factory to create samples from real files
factory1 = SampleFactory(cuckoo=None,
base_dir=None,
job_hash_regex=None,
keep_mail_data=False,
processing_info_dir=None)
# sampe factory to create samples with defined content
factory2 = CreatingSampleFactory(cuckoo=None,
base_dir=None,
job_hash_regex=None,
keep_mail_data=False,
processing_info_dir=None)
tests_data_dir = os.path.dirname(
os.path.abspath(__file__)) + "/test-data"
combinations = [
# no office document file extension
[Result.unknown,
factory2.make_sample('test.nodoc', 'test')],
# test with empty file
[
Result.unknown,
factory1.make_sample(tests_data_dir + '/office/empty.doc')
],
# office document with 'suspicious' in macro code
[
Result.bad,
factory1.make_sample(tests_data_dir +
'/office/suspiciousMacro.doc')
],
# test with blank word doc
[
Result.unknown,
factory1.make_sample(tests_data_dir + '/office/blank.doc')
],
# test with legitimate macro
[
Result.unknown,
factory1.make_sample(tests_data_dir + '/office/legitmacro.xls')
]
]
for expected, sample in combinations:
result = rule.evaluate(sample)
self.assertEqual(result.result, expected)
# test if macro present
rule = OfficeMacroRule(CreatingConfigParser(config))
combinations = [
# no office document file extension
[Result.unknown,
factory2.make_sample('test.nodoc', 'test')],
# test with empty file
[
Result.unknown,
factory1.make_sample(tests_data_dir + '/office/empty.doc')
],
# office document with 'suspicious' in macro code
[
Result.bad,
factory1.make_sample(tests_data_dir +
'/office/suspiciousMacro.doc')
],
# test with blank word doc
[
Result.unknown,
factory1.make_sample(tests_data_dir + '/office/blank.doc')
],
# test with legitimate macro
[
Result.bad,
factory1.make_sample(tests_data_dir + '/office/legitmacro.xls')
]
]
for expected, sample in combinations:
result = rule.evaluate(sample)
self.assertEqual(result.result, expected)
def run():
""" Runs the Peekaboo daemon. """
arg_parser = ArgumentParser(
description='Peekaboo Extended Email Attachment Behavior Observation Owl'
)
arg_parser.add_argument(
'-c', '--config',
action='store',
help='The configuration file for Peekaboo.'
)
arg_parser.add_argument(
'-d', '--debug',
action='store_true',
help="Run Peekaboo in debug mode regardless of what's specified in the configuration."
)
arg_parser.add_argument(
'-D', '--daemon',
action='store_true',
help='Run Peekaboo in daemon mode (suppresses the logo to be written to STDOUT).'
)
args = arg_parser.parse_args()
print('Starting Peekaboo %s.' % __version__)
if not args.daemon:
print(PEEKABOO_OWL)
# Check if CLI arguments override the configuration
log_level = None
if args.debug:
log_level = logging.DEBUG
try:
config = PeekabooConfig(config_file=args.config, log_level=log_level)
logger.debug(config)
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# find localisation in our package directory
locale_domain = 'peekaboo'
locale_dir = os.path.join(os.path.dirname(__file__), 'locale')
languages = None
if config.report_locale:
logger.debug('Looking for translations for preconfigured locale "%s"',
config.report_locale)
languages = [config.report_locale]
if not gettext.find(locale_domain, locale_dir, languages):
logger.warning('Translation file not found - falling back to '
'system configuration.')
languages = None
logger.debug('Installing report message translations')
translation = gettext.translation(locale_domain, locale_dir, languages,
fallback=True)
translation.install()
# establish a connection to the database
try:
db_con = PeekabooDatabase(
db_url=config.db_url, instance_id=config.cluster_instance_id,
stale_in_flight_threshold=config.cluster_stale_in_flight_threshold,
log_level=config.db_log_level)
except PeekabooDatabaseError as error:
logging.critical(error)
sys.exit(1)
except SQLAlchemyError as dberr:
logger.critical('Failed to establish a connection to the database '
'at %s: %s', config.db_url, dberr)
sys.exit(1)
# initialize the daemon infrastructure such as PID file and dropping
# privileges, automatically cleans up after itself when going out of scope
daemon_infrastructure = PeekabooDaemonInfrastructure(
config.pid_file, config.sock_file, config.user, config.group)
daemon_infrastructure.init()
# clear all our in flight samples and all instances' stale in flight
# samples
db_con.clear_in_flight_samples()
db_con.clear_stale_in_flight_samples()
# a cluster duplicate interval of 0 disables the handler thread which is
# what we want if we don't have an instance_id and therefore are alone
cldup_check_interval = 0
if config.cluster_instance_id > 0:
cldup_check_interval = config.cluster_duplicate_check_interval
if cldup_check_interval < 5:
cldup_check_interval = 5
logger.warning("Raising excessively low cluster duplicate check "
"interval to %d seconds.",
cldup_check_interval)
sig_handler = SignalHandler()
# read in the analyzer and ruleset configuration and start the job queue
try:
ruleset_config = PeekabooConfigParser(config.ruleset_config)
analyzer_config = PeekabooAnalyzerConfig(config.analyzer_config)
job_queue = JobQueue(
worker_count=config.worker_count, ruleset_config=ruleset_config,
db_con=db_con, analyzer_config=analyzer_config,
cluster_duplicate_check_interval=cldup_check_interval)
sig_handler.register_listener(job_queue)
job_queue.start()
except PeekabooConfigException as error:
logging.critical(error)
sys.exit(1)
# Factory producing almost identical samples providing them with global
# config values and references to other objects they need, such as database
# connection and connection map.
sample_factory = SampleFactory(
config.sample_base_dir, config.job_hash_regex,
config.keep_mail_data, config.processing_info_dir)
# We only want to accept 2 * worker_count connections.
try:
server = PeekabooServer(
sock_file=config.sock_file, job_queue=job_queue,
sample_factory=sample_factory,
request_queue_size=config.worker_count * 2,
sock_group=config.sock_group,
sock_mode=config.sock_mode)
except Exception as error:
logger.critical('Failed to start Peekaboo Server: %s', error)
job_queue.shut_down()
job_queue.close_down()
sys.exit(1)
# abort startup if shutdown was requested meanwhile
if sig_handler.shutdown_requested:
sys.exit(0)
sig_handler.register_listener(server)
SystemdNotifier().notify("READY=1")
server.serve()
# trigger shutdowns of other components (if not already ongoing triggered
# by e.g. the signal handler), server will already be shut down at this
# point signaled by the fact that serve() above returned
job_queue.shut_down()
# close down components after they've shut down
job_queue.close_down()
# do a final cleanup pass through the database
try:
db_con.clear_in_flight_samples()
db_con.clear_stale_in_flight_samples()
except PeekabooDatabaseError as dberr:
logger.error(dberr)
sys.exit(0)