def setup_scheduler():
"""Sets up the APScheduler"""
log = logging.getLogger('apscheduler')
try:
accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa
accounts = [account.name for account in accounts]
for account in accounts:
print "Scheduler adding account {}".format(account)
rep = Reporter(accounts=[account])
for period in rep.get_intervals(account):
scheduler.add_interval_job(
run_change_reporter,
minutes=period,
start_date=datetime.now()+timedelta(seconds=2),
args=[account, period]
)
auditors = [a for (_, a) in rep.get_watchauditors(account) if a]
if auditors:
scheduler.add_cron_job(_audit_changes, hour=10, day_of_week="mon-fri", args=[account, auditors, True])
# Clear out old exceptions:
scheduler.add_cron_job(_clear_old_exceptions, hour=3, minute=0)
except Exception as e:
app.logger.warn("Scheduler Exception: {}".format(e))
app.logger.warn(traceback.format_exc())
store_exception("scheduler", None, e)
def setup_scheduler():
"""Sets up the APScheduler"""
log = logging.getLogger('apscheduler')
try:
accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa
accounts = [account.name for account in accounts]
for account in accounts:
app.logger.debug("Scheduler adding account {}".format(account))
rep = Reporter(account=account)
delay = app.config.get('REPORTER_START_DELAY', 10)
for period in rep.get_intervals(account):
scheduler.add_interval_job(
run_change_reporter,
minutes=period,
start_date=datetime.now()+timedelta(seconds=delay),
args=[[account], period]
)
auditors = []
for monitor in all_monitors(account):
auditors.extend(monitor.auditors)
scheduler.add_cron_job(_audit_changes, hour=10, day_of_week="mon-fri", args=[account, auditors, True])
# Clear out old exceptions:
scheduler.add_cron_job(_clear_old_exceptions, hour=3, minute=0)
except Exception as e:
app.logger.warn("Scheduler Exception: {}".format(e))
app.logger.warn(traceback.format_exc())
store_exception("scheduler", None, e)
def setup_scheduler():
"""Sets up the APScheduler"""
log = logging.getLogger('apscheduler')
try:
accounts = Account.query.filter(Account.third_party == False).filter(
Account.active == True).all() # noqa
accounts = [account.name for account in accounts]
for account in accounts:
app.logger.debug("Scheduler adding account {}".format(account))
rep = Reporter(account=account)
delay = app.config.get('REPORTER_START_DELAY', 10)
for period in rep.get_intervals(account):
scheduler.add_interval_job(run_change_reporter,
minutes=period,
start_date=datetime.now() +
timedelta(seconds=delay),
args=[[account], period])
auditors = []
for monitor in all_monitors(account):
auditors.extend(monitor.auditors)
scheduler.add_cron_job(_audit_changes,
hour=10,
day_of_week="mon-fri",
args=[account, auditors, True])
# Clear out old exceptions:
scheduler.add_cron_job(_clear_old_exceptions, hour=3, minute=0)
except Exception as e:
app.logger.warn("Scheduler Exception: {}".format(e))
app.logger.warn(traceback.format_exc())
store_exception("scheduler", None, e)
def test_run_with_interval_watcher_dependencies(self):
"""
If an interval is passed to reporter.run(), the reporter will run all watchers in the interval
along with their auditors. It will also reaudit all existing items of watchers that are not in
the interval but are dependent on watchers/auditors in the interval. This is done because any
changes to the dependencies could change the audit results even is the items have not changed.
In this case, index1 and index2 are in the interval and index3 is dependent on index1 watcher.
Expected result:
Watchers of index1 and index2 are run
New items of index1 and index2 are audited
Items of index3 are reaudited
"""
from security_monkey.reporter import Reporter
build_mock_result(watcher_configs, auditor_configs_with_watcher_dependencies)
reporter = Reporter(account="TEST_ACCOUNT")
reporter.run("TEST_ACCOUNT", 15)
watcher_keys = RUNTIME_WATCHERS.keys()
self.assertEqual(first=2, second=len(watcher_keys),
msg="Should run 2 watchers but ran {}"
.format(len(watcher_keys)))
self.assertTrue('index1' in watcher_keys,
msg="Watcher index1 not run")
self.assertTrue('index2' in watcher_keys,
msg="Watcher index2 not run")
self.assertEqual(first=1, second=len(RUNTIME_WATCHERS['index1']),
msg="Watcher index1 should have audited 1 item but audited {}"
.format(len(RUNTIME_WATCHERS['index1'])))
self.assertEqual(first=1, second=len(RUNTIME_WATCHERS['index2']),
msg="Watcher index2 should have audited 1 item but audited {}"
.format(len(RUNTIME_WATCHERS['index2'])))
auditor_keys = RUNTIME_AUDIT_COUNTS.keys()
self.assertEqual(first=3, second=len(auditor_keys),
msg="Should run 3 auditors but ran {}"
.format(len(auditor_keys)))
self.assertTrue('index1' in auditor_keys,
msg="Auditor index1 not run")
self.assertTrue('index2' in auditor_keys,
msg="Auditor index2 not run")
self.assertTrue('index3' in auditor_keys,
msg="Auditor index3 not run")
self.assertEqual(first=1, second=RUNTIME_AUDIT_COUNTS['index1'],
msg="Auditor index1 should run once but ran {} times"
.format(RUNTIME_AUDIT_COUNTS['index1']))
self.assertEqual(first=1, second=RUNTIME_AUDIT_COUNTS['index2'],
msg="Auditor index2 should run once but ran {} times"
.format(RUNTIME_AUDIT_COUNTS['index2']))
self.assertEqual(first=1, second=RUNTIME_AUDIT_COUNTS['index3'],
msg="Auditor index3 should run once but ran {} times"
.format(RUNTIME_AUDIT_COUNTS['index3']))
def run_change_reporter(account_names, interval=None):
""" Runs Reporter """
try:
for account in account_names:
reporter = Reporter(account=account, debug=True)
reporter.run(account, interval)
except (OperationalError, InvalidRequestError, StatementError) as e:
app.logger.exception("Database error processing accounts %s, cleaning up session.", account_names)
db.session.remove()
store_exception("scheduler-run-change-reporter", None, e)
def run_change_reporter(account_names, interval=None):
""" Runs Reporter """
try:
for account in account_names:
reporter = Reporter(account=account, alert_accounts=account_names, debug=True)
reporter.run(account, interval)
except (OperationalError, InvalidRequestError, StatementError) as e:
app.logger.exception("Database error processing accounts %s, cleaning up session.", account_names)
db.session.remove()
store_exception("scheduler-run-change-reporter", None, e)
def run_change_reporter(accounts, interval=None):
""" Runs Reporter """
try:
accounts = __prep_accounts__(accounts)
reporter = Reporter(accounts=accounts, alert_accounts=accounts, debug=True)
for account in accounts:
reporter.run(account, interval)
except (OperationalError, InvalidRequestError, StatementError):
app.logger.exception("Database error processing accounts %s, cleaning up session.", accounts)
db.session.remove()
def run_change_reporter(accounts, interval=None):
""" Runs Reporter """
try:
accounts = __prep_accounts__(accounts)
reporter = Reporter(accounts=accounts,
alert_accounts=accounts,
debug=True)
for account in accounts:
reporter.run(account, interval)
except (OperationalError, InvalidRequestError, StatementError):
app.logger.exception(
"Database error processing accounts %s, cleaning up session.",
accounts)
db.session.remove()
def manual_run_change_reporter(accounts):
"""Manual change reporting from the command line"""
app.logger.info("[ ] Executing manual change reporter task...")
try:
for account in accounts:
time1 = time.time()
rep = Reporter(account=account)
for monitor in rep.all_monitors:
if monitor.watcher:
app.logger.info("[ ] Running change finder for "
"account: {} technology: {}".format(
account, monitor.watcher.index))
reporter_logic(account, monitor.watcher.index)
time2 = time.time()
app.logger.info('[@] Run Account %s took %0.1f s' %
(account, (time2 - time1)))
app.logger.info("[+] Completed manual change reporting.")
except (OperationalError, InvalidRequestError, StatementError) as e:
app.logger.exception(
"[X] Database error processing cleaning up session.")
db.session.remove()
store_exception("scheduler-run-change-reporter", None, e)
raise e
def run_change_reporter(accounts):
""" Runs Reporter """
accounts = __prep_accounts__(accounts)
reporter = Reporter(accounts=accounts, alert_accounts=accounts, debug=True)
for account in accounts:
reporter.run(account)
def setup_the_tasks(sender, **kwargs):
setup()
# Purge out all current tasks waiting to execute:
purge_it()
# Get the celery configuration (Get the raw module since Celery doesn't document a good way to do this
# see https://github.com/celery/celery/issues/4633):
celery_config = get_celery_config_file()
# Add all the tasks:
try:
accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa
for account in accounts:
rep = Reporter(account=account.name)
# Is this a dedicated watcher stack, or is this stack ignoring anything?
only_watch = get_sm_celery_config_value(celery_config, "security_monkey_only_watch", set)
# If only_watch is set, then ignoring is ignored.
if only_watch:
ignoring = set()
else:
# Check if we are ignoring any watchers:
ignoring = get_sm_celery_config_value(celery_config, "security_monkey_watcher_ignore", set) or set()
for monitor in rep.all_monitors:
# Is this watcher enabled?
if monitor.watcher.is_active() and monitor.watcher.index not in ignoring:
# Did we specify specific watchers to run?
if only_watch and monitor.watcher.index not in only_watch:
continue
app.logger.info("[ ] Scheduling tasks for {type} account: {name}".format(type=account.type.name,
name=account.name))
interval = monitor.watcher.get_interval()
if not interval:
app.logger.debug("[/] Skipping watcher for technology: {} because it is set for external "
"monitoring.".format(monitor.watcher.index))
continue
app.logger.debug("[{}] Scheduling for technology: {}".format(account.type.name,
monitor.watcher.index))
# Start the task immediately:
task_account_tech.apply_async((account.name, monitor.watcher.index))
app.logger.debug("[-->] Scheduled immediate task")
schedule = interval * 60
schedule_at_full_hour = get_sm_celery_config_value(celery_config, "schedule_at_full_hour", bool) or False
if schedule_at_full_hour:
if interval == 15: # 15 minute
schedule = crontab(minute="0,15,30,45")
elif interval == 60: # Hourly
schedule = crontab(minute="0")
elif interval == 720: # 12 hour
schedule = crontab(minute="0", hour="0,12")
elif interval == 1440: # Daily
schedule = crontab(minute="0", hour="0")
elif interval == 10080: # Weekly
schedule = crontab(minute="0", hour="0", day_of_week="0")
# Schedule it based on the schedule:
sender.add_periodic_task(schedule, task_account_tech.s(account.name, monitor.watcher.index))
app.logger.debug("[+] Scheduled task to occur every {} minutes".format(interval))
# TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily
# disabled this to avoid many duplicate events from getting added.
# Also schedule a manual audit changer just in case it doesn't properly
# audit (only for non-batched):
# if not monitor.batch_support:
# sender.add_periodic_task(
# crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index))
# app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index))
#
# app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name,
# monitor.watcher.index))
app.logger.debug("[+] Completed scheduling tasks for account: {}".format(account.name))
# Schedule the task for clearing out old exceptions:
app.logger.info("Scheduling task to clear out old exceptions.")
# Run every 24 hours (and clear it now):
clear_expired_exceptions.apply_async()
sender.add_periodic_task(86400, clear_expired_exceptions.s())
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Scheduler Exception: {}".format(e))
app.logger.error(traceback.format_exc())
store_exception("scheduler", None, e)
def run_change_reporter(accounts):
""" Runs Reporter """
accounts = __prep_accounts__(accounts)
reporter = Reporter(accounts=accounts, alert_accounts=accounts, debug=True)
for account in accounts:
reporter.run(account)
def test_report_batch_changes(self):
from security_monkey.alerter import Alerter
from security_monkey.reporter import Reporter
from security_monkey.datastore import Item, ItemRevision, ItemAudit
from security_monkey.monitors import Monitor
from security_monkey.watchers.iam.iam_role import IAMRole
from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
account_type_result = AccountType.query.filter(
AccountType.name == "AWS").one()
db.session.add(account_type_result)
db.session.commit()
test_account = Account(identifier="012345678910",
name="TEST_ACCOUNT",
account_type_id=account_type_result.id,
notes="TEST_ACCOUNT1",
third_party=False,
active=True)
watcher = IAMRole(accounts=[test_account.name])
db.session.commit()
watcher.batched_size = 3 # should loop 4 times
self.add_roles()
# Set up the monitor:
batched_monitor = Monitor(IAMRole, test_account)
batched_monitor.watcher = watcher
batched_monitor.auditors = [
IAMRoleAuditor(accounts=[test_account.name])
]
# Set up the Reporter:
import security_monkey.reporter
old_all_monitors = security_monkey.reporter.all_monitors
security_monkey.reporter.all_monitors = lambda x, y: []
test_reporter = Reporter()
test_reporter.all_monitors = [batched_monitor]
test_reporter.account_alerter = Alerter(
watchers_auditors=test_reporter.all_monitors,
account=test_account.name)
import security_monkey.scheduler
# import security_monkey.monitors
# old_get_monitors = security_monkey.scheduler.get_monitors
security_monkey.scheduler.get_monitors = lambda x, y, z: [
batched_monitor
]
# Moto screws up the IAM Role ARN -- so we need to fix it:
original_slurp_list = watcher.slurp_list
original_slurp = watcher.slurp
def mock_slurp_list():
items, exception_map = original_slurp_list()
for item in watcher.total_list:
item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(
item["RoleName"])
return items, exception_map
def mock_slurp():
batched_items, exception_map = original_slurp()
for item in batched_items:
item.arn = "arn:aws:iam::012345678910:role/{}".format(
item.name)
item.config["Arn"] = item.arn
item.config["RoleId"] = item.name # Need this to stay the same
return batched_items, exception_map
watcher.slurp_list = mock_slurp_list
watcher.slurp = mock_slurp
test_reporter.run(account=test_account.name)
# Check that all items were added to the DB:
assert len(Item.query.all()) == 11
# Check that we have exactly 11 item revisions:
assert len(ItemRevision.query.all()) == 11
# Check that there are audit issues for all 11 items:
assert len(ItemAudit.query.all()) == 11
mock_iam().stop()
mock_sts().stop()
# Something isn't cleaning itself up properly and causing other core tests to fail.
# This is the solution:
security_monkey.reporter.all_monitors = old_all_monitors
import monitor_mock
security_monkey.scheduler.get_monitors = monitor_mock.mock_get_monitors
def test_report_batch_changes(self):
from security_monkey.alerter import Alerter
from security_monkey.reporter import Reporter
from security_monkey.datastore import Item, ItemRevision, ItemAudit
from security_monkey.monitors import Monitor
from security_monkey.watchers.iam.iam_role import IAMRole
from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
account_type_result = AccountType.query.filter(AccountType.name == "AWS").one()
db.session.add(account_type_result)
db.session.commit()
test_account = Account(name="TEST_ACCOUNT")
watcher = IAMRole(accounts=[test_account.name])
db.session.commit()
watcher.batched_size = 3 # should loop 4 times
self.add_roles()
# Set up the monitor:
batched_monitor = Monitor(IAMRole, test_account)
batched_monitor.watcher = watcher
batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
# Set up the Reporter:
import security_monkey.reporter
old_all_monitors = security_monkey.reporter.all_monitors
security_monkey.reporter.all_monitors = lambda x, y: []
test_reporter = Reporter()
test_reporter.all_monitors = [batched_monitor]
test_reporter.account_alerter = Alerter(watchers_auditors=test_reporter.all_monitors, account=test_account.name)
import security_monkey.scheduler
# import security_monkey.monitors
# old_get_monitors = security_monkey.scheduler.get_monitors
security_monkey.scheduler.get_monitors = lambda x, y, z: [batched_monitor]
# Moto screws up the IAM Role ARN -- so we need to fix it:
original_slurp_list = watcher.slurp_list
original_slurp = watcher.slurp
def mock_slurp_list():
exception_map = original_slurp_list()
for item in watcher.total_list:
item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(item["RoleName"])
return exception_map
def mock_slurp():
batched_items, exception_map = original_slurp()
for item in batched_items:
item.arn = "arn:aws:iam::012345678910:role/{}".format(item.name)
item.config["Arn"] = item.arn
item.config["RoleId"] = item.name # Need this to stay the same
return batched_items, exception_map
watcher.slurp_list = mock_slurp_list
watcher.slurp = mock_slurp
test_reporter.run(account=test_account.name)
# Check that all items were added to the DB:
assert len(Item.query.all()) == 11
# Check that we have exactly 11 item revisions:
assert len(ItemRevision.query.all()) == 11
# Check that there are audit issues for all 11 items:
assert len(ItemAudit.query.all()) == 11
mock_iam().stop()
mock_sts().stop()
# Something isn't cleaning itself up properly and causing other core tests to fail.
# This is the solution:
security_monkey.reporter.all_monitors = old_all_monitors
import monitor_mock
security_monkey.scheduler.get_monitors = monitor_mock.mock_get_monitors
def setup_the_tasks(sender, **kwargs):
setup()
# Purge out all current tasks waiting to execute:
purge_it()
# Add all the tasks:
try:
# TODO: Investigate options to have the scheduler skip different types of accounts
accounts = Account.query.filter(Account.third_party == False).filter(
Account.active == True).all() # noqa
for account in accounts:
app.logger.info(
"[ ] Scheduling tasks for {type} account: {name}".format(
type=account.type.name, name=account.name))
rep = Reporter(account=account.name)
for monitor in rep.all_monitors:
if monitor.watcher:
app.logger.debug(
"[{}] Scheduling for technology: {}".format(
account.type.name, monitor.watcher.index))
interval = monitor.watcher.get_interval() * 60
# Start the task immediately:
task_account_tech.apply_async(
(account.name, monitor.watcher.index))
app.logger.debug("[-->] Scheduled immediate task")
# Schedule it based on the schedule:
sender.add_periodic_task(
interval,
task_account_tech.s(account.name,
monitor.watcher.index))
app.logger.debug(
"[+] Scheduled task to occur every {} minutes".format(
interval))
# TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily
# disabled this to avoid many duplicate events from getting added.
# Also schedule a manual audit changer just in case it doesn't properly
# audit (only for non-batched):
# if not monitor.batch_support:
# sender.add_periodic_task(
# crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index))
# app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index))
#
# app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name,
# monitor.watcher.index))
app.logger.debug(
"[+] Completed scheduling tasks for account: {}".format(
account.name))
# Schedule the task for clearing out old exceptions:
app.logger.info("Scheduling task to clear out old exceptions.")
# TODO: Investigate if this creates many duplicate tasks RE: Celery bug mentioned above
sender.add_periodic_task(crontab(hour=3, minute=0),
clear_expired_exceptions.s())
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Scheduler Exception: {}".format(e))
app.logger.error(traceback.format_exc())
store_exception("scheduler", None, e)
