def task_account_tech(self, account_name, technology_name):
setup()
app.logger.info(
"[ ] Executing Celery task for account: {}, technology: {}".format(
account_name, technology_name))
time1 = time.time()
try:
reporter_logic(account_name, technology_name)
time2 = time.time()
app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' %
(account_name, technology_name, (time2 - time1)))
app.logger.info(
"[+] Completed Celery task for account: {}, technology: {}".format(
account_name, technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error(
"[X] Task Account Scheduler Exception ({}/{}): {}".format(
account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-watch", None, e)
raise self.retry(exc=e)
def task_audit(self, account_name, technology_name):
setup()
app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
technology_name))
# Verify that the account exists (was it deleted? was it renamed?):
if not Account.query.filter(Account.name == account_name).first():
app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format(
account_name
))
return
try:
audit_changes([account_name], [technology_name], True)
app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-audit", None, e)
self.retry(exc=e)
def decorated_function(*args, **kwargs):
# prevent these from being passed to the wrapped function:
m = kwargs.pop if pop_exception_fields else kwargs.get
exception_values = {
'index': m('index'),
'account': m('account_name', None),
'exception_record_region': m('exception_record_region', None),
'name': m('name', None),
'exception_map': m('exception_map', {})
}
try:
return f(*args, **kwargs)
except Exception as e:
if sentry:
sentry.captureException()
index = exception_values['index']
account = exception_values['account']
# Allow the recording region to be overridden for universal tech like IAM
region = exception_values['exception_record_region'] or kwargs.get('region')
name = exception_values['name']
exception_map = exception_values['exception_map']
exc = BotoConnectionIssue(str(e), index, account, name)
if name:
location = (index, account, region, name)
elif region:
location = (index, account, region)
elif account:
location = (index, account)
else:
location = (index, )
exception_map[location] = exc
# Store the exception (the original one passed in, not exc):
store_exception(source=source, location=location, exception=e)
def task_account_tech(self, account_name, technology_name):
setup()
app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
time1 = time.time()
# Verify that the account exists (was it deleted? was it renamed?):
if not Account.query.filter(Account.name == account_name).first():
app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format(
account_name
))
return
try:
reporter_logic(account_name, technology_name)
time2 = time.time()
app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
technology_name, (time2 - time1)))
app.logger.info(
"[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-watch", None, e)
raise self.retry(exc=e)
def task_audit(self, account_name, technology_name):
setup()
app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
technology_name))
# Verify that the account exists (was it deleted? was it renamed?):
if not Account.query.filter(Account.name == account_name).first():
app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format(
account_name
))
return
try:
audit_changes([account_name], [technology_name], True)
app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-audit", None, e)
self.retry(exc=e)
def task_account_tech(self, account_name, technology_name):
setup()
app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
time1 = time.time()
# Verify that the account exists (was it deleted? was it renamed?):
if not Account.query.filter(Account.name == account_name).first():
app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format(
account_name
))
return
try:
reporter_logic(account_name, technology_name)
time2 = time.time()
app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
technology_name, (time2 - time1)))
app.logger.info(
"[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-watch", None, e)
raise self.retry(exc=e)
def task_audit(self, account_name, technology_name):
setup()
app.logger.info(
"[ ] Executing Celery task to audit changes for Account: {} Technology: {}"
.format(account_name, technology_name))
try:
audit_changes([account_name], [technology_name], True)
app.logger.info(
"[+] Completed Celery task for account: {}, technology: {}".format(
account_name, technology_name))
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error(
"[X] Task Audit Scheduler Exception ({}/{}): {}".format(
account_name, technology_name, e))
app.logger.error(traceback.format_exc())
store_exception("scheduler-exception-on-audit", None, e)
self.retry(exc=e)
def decorated_function(*args, **kwargs):
# prevent these from being passed to the wrapped function:
m = kwargs.pop if pop_exception_fields else kwargs.get
exception_values = {
'index': m('index'),
'account': m('account_name', None),
'exception_record_region': m('exception_record_region', None),
'name': m('name', None),
'exception_map': m('exception_map')
}
try:
return f(*args, **kwargs)
except Exception as e:
if sentry:
sentry.captureException()
index = exception_values['index']
account = exception_values['account']
# Allow the recording region to be overridden for universal tech like IAM
region = exception_values[
'exception_record_region'] or kwargs.get('region')
name = exception_values['name']
exception_map = exception_values['exception_map']
exc = BotoConnectionIssue(str(e), index, account, name)
if name:
location = (index, account, region, name)
elif region:
location = (index, account, region)
elif account:
location = (index, account)
else:
location = (index, )
exception_map[location] = exc
# Store the exception (the original one passed in, not exc):
store_exception(source=source, location=location, exception=e)
def setup_the_tasks(sender, **kwargs):
setup()
# Purge out all current tasks waiting to execute:
purge_it()
# Get the celery configuration (Get the raw module since Celery doesn't document a good way to do this
# see https://github.com/celery/celery/issues/4633):
celery_config = get_celery_config_file()
# Add all the tasks:
try:
accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa
for account in accounts:
rep = Reporter(account=account.name)
# Is this a dedicated watcher stack, or is this stack ignoring anything?
only_watch = get_sm_celery_config_value(celery_config, "security_monkey_only_watch", set)
# If only_watch is set, then ignoring is ignored.
if only_watch:
ignoring = set()
else:
# Check if we are ignoring any watchers:
ignoring = get_sm_celery_config_value(celery_config, "security_monkey_watcher_ignore", set) or set()
for monitor in rep.all_monitors:
# Is this watcher enabled?
if monitor.watcher.is_active() and monitor.watcher.index not in ignoring:
# Did we specify specific watchers to run?
if only_watch and monitor.watcher.index not in only_watch:
continue
app.logger.info("[ ] Scheduling tasks for {type} account: {name}".format(type=account.type.name,
name=account.name))
interval = monitor.watcher.get_interval()
if not interval:
app.logger.debug("[/] Skipping watcher for technology: {} because it is set for external "
"monitoring.".format(monitor.watcher.index))
continue
app.logger.debug("[{}] Scheduling for technology: {}".format(account.type.name,
monitor.watcher.index))
# Start the task immediately:
task_account_tech.apply_async((account.name, monitor.watcher.index))
app.logger.debug("[-->] Scheduled immediate task")
schedule = interval * 60
schedule_at_full_hour = get_sm_celery_config_value(celery_config, "schedule_at_full_hour", bool) or False
if schedule_at_full_hour:
if interval == 15: # 15 minute
schedule = crontab(minute="0,15,30,45")
elif interval == 60: # Hourly
schedule = crontab(minute="0")
elif interval == 720: # 12 hour
schedule = crontab(minute="0", hour="0,12")
elif interval == 1440: # Daily
schedule = crontab(minute="0", hour="0")
elif interval == 10080: # Weekly
schedule = crontab(minute="0", hour="0", day_of_week="0")
# Schedule it based on the schedule:
sender.add_periodic_task(schedule, task_account_tech.s(account.name, monitor.watcher.index))
app.logger.debug("[+] Scheduled task to occur every {} minutes".format(interval))
# TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily
# disabled this to avoid many duplicate events from getting added.
# Also schedule a manual audit changer just in case it doesn't properly
# audit (only for non-batched):
# if not monitor.batch_support:
# sender.add_periodic_task(
# crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index))
# app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index))
#
# app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name,
# monitor.watcher.index))
app.logger.debug("[+] Completed scheduling tasks for account: {}".format(account.name))
# Schedule the task for clearing out old exceptions:
app.logger.info("Scheduling task to clear out old exceptions.")
# Run every 24 hours (and clear it now):
clear_expired_exceptions.apply_async()
sender.add_periodic_task(86400, clear_expired_exceptions.s())
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Scheduler Exception: {}".format(e))
app.logger.error(traceback.format_exc())
store_exception("scheduler", None, e)
def setup_the_tasks(sender, **kwargs):
setup()
# Purge out all current tasks waiting to execute:
purge_it()
# Get the celery configuration (Get the raw module since Celery doesn't document a good way to do this
# see https://github.com/celery/celery/issues/4633):
celery_config = get_celery_config_file()
# Add all the tasks:
try:
accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa
for account in accounts:
rep = Reporter(account=account.name)
# Is this a dedicated watcher stack, or is this stack ignoring anything?
only_watch = get_sm_celery_config_value(celery_config, "security_monkey_only_watch", set)
# If only_watch is set, then ignoring is ignored.
if only_watch:
ignoring = set()
else:
# Check if we are ignoring any watchers:
ignoring = get_sm_celery_config_value(celery_config, "security_monkey_watcher_ignore", set) or set()
for monitor in rep.all_monitors:
# Is this watcher enabled?
if monitor.watcher.is_active() and monitor.watcher.index not in ignoring:
# Did we specify specific watchers to run?
if only_watch and monitor.watcher.index not in only_watch:
continue
app.logger.info("[ ] Scheduling tasks for {type} account: {name}".format(type=account.type.name,
name=account.name))
interval = monitor.watcher.get_interval() * 60
if not interval:
app.logger.debug("[/] Skipping watcher for technology: {} because it is set for external "
"monitoring.".format(monitor.watcher.index))
continue
app.logger.debug("[{}] Scheduling for technology: {}".format(account.type.name,
monitor.watcher.index))
# Start the task immediately:
task_account_tech.apply_async((account.name, monitor.watcher.index))
app.logger.debug("[-->] Scheduled immediate task")
# Schedule it based on the schedule:
sender.add_periodic_task(interval, task_account_tech.s(account.name, monitor.watcher.index))
app.logger.debug("[+] Scheduled task to occur every {} minutes".format(interval))
# TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily
# disabled this to avoid many duplicate events from getting added.
# Also schedule a manual audit changer just in case it doesn't properly
# audit (only for non-batched):
# if not monitor.batch_support:
# sender.add_periodic_task(
# crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index))
# app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index))
#
# app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name,
# monitor.watcher.index))
app.logger.debug("[+] Completed scheduling tasks for account: {}".format(account.name))
# Schedule the task for clearing out old exceptions:
app.logger.info("Scheduling task to clear out old exceptions.")
# Run every 24 hours (and clear it now):
clear_expired_exceptions.apply_async()
sender.add_periodic_task(86400, clear_expired_exceptions.s())
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Scheduler Exception: {}".format(e))
app.logger.error(traceback.format_exc())
store_exception("scheduler", None, e)
def setup_the_tasks(sender, **kwargs):
setup()
# Purge out all current tasks waiting to execute:
purge_it()
# Add all the tasks:
try:
# TODO: Investigate options to have the scheduler skip different types of accounts
accounts = Account.query.filter(Account.third_party == False).filter(
Account.active == True).all() # noqa
for account in accounts:
app.logger.info(
"[ ] Scheduling tasks for {type} account: {name}".format(
type=account.type.name, name=account.name))
rep = Reporter(account=account.name)
for monitor in rep.all_monitors:
if monitor.watcher:
app.logger.debug(
"[{}] Scheduling for technology: {}".format(
account.type.name, monitor.watcher.index))
interval = monitor.watcher.get_interval() * 60
# Start the task immediately:
task_account_tech.apply_async(
(account.name, monitor.watcher.index))
app.logger.debug("[-->] Scheduled immediate task")
# Schedule it based on the schedule:
sender.add_periodic_task(
interval,
task_account_tech.s(account.name,
monitor.watcher.index))
app.logger.debug(
"[+] Scheduled task to occur every {} minutes".format(
interval))
# TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily
# disabled this to avoid many duplicate events from getting added.
# Also schedule a manual audit changer just in case it doesn't properly
# audit (only for non-batched):
# if not monitor.batch_support:
# sender.add_periodic_task(
# crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index))
# app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index))
#
# app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name,
# monitor.watcher.index))
app.logger.debug(
"[+] Completed scheduling tasks for account: {}".format(
account.name))
# Schedule the task for clearing out old exceptions:
app.logger.info("Scheduling task to clear out old exceptions.")
# TODO: Investigate if this creates many duplicate tasks RE: Celery bug mentioned above
sender.add_periodic_task(crontab(hour=3, minute=0),
clear_expired_exceptions.s())
except Exception as e:
if sentry:
sentry.captureException()
app.logger.error("[X] Scheduler Exception: {}".format(e))
app.logger.error(traceback.format_exc())
store_exception("scheduler", None, e)
