def task_account_tech(self, account_name, technology_name): setup() app.logger.info( "[ ] Executing Celery task for account: {}, technology: {}".format( account_name, technology_name)) time1 = time.time() try: reporter_logic(account_name, technology_name) time2 = time.time() app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name, technology_name, (time2 - time1))) app.logger.info( "[+] Completed Celery task for account: {}, technology: {}".format( account_name, technology_name)) except Exception as e: if sentry: sentry.captureException() app.logger.error( "[X] Task Account Scheduler Exception ({}/{}): {}".format( account_name, technology_name, e)) app.logger.error(traceback.format_exc()) store_exception("scheduler-exception-on-watch", None, e) raise self.retry(exc=e)
def task_audit(self, account_name, technology_name): setup() app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name, technology_name)) # Verify that the account exists (was it deleted? was it renamed?): if not Account.query.filter(Account.name == account_name).first(): app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format( account_name )) return try: audit_changes([account_name], [technology_name], True) app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name)) except Exception as e: if sentry: sentry.captureException() app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e)) app.logger.error(traceback.format_exc()) store_exception("scheduler-exception-on-audit", None, e) self.retry(exc=e)
def decorated_function(*args, **kwargs): # prevent these from being passed to the wrapped function: m = kwargs.pop if pop_exception_fields else kwargs.get exception_values = { 'index': m('index'), 'account': m('account_name', None), 'exception_record_region': m('exception_record_region', None), 'name': m('name', None), 'exception_map': m('exception_map', {}) } try: return f(*args, **kwargs) except Exception as e: if sentry: sentry.captureException() index = exception_values['index'] account = exception_values['account'] # Allow the recording region to be overridden for universal tech like IAM region = exception_values['exception_record_region'] or kwargs.get('region') name = exception_values['name'] exception_map = exception_values['exception_map'] exc = BotoConnectionIssue(str(e), index, account, name) if name: location = (index, account, region, name) elif region: location = (index, account, region) elif account: location = (index, account) else: location = (index, ) exception_map[location] = exc # Store the exception (the original one passed in, not exc): store_exception(source=source, location=location, exception=e)
def task_account_tech(self, account_name, technology_name): setup() app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name)) time1 = time.time() # Verify that the account exists (was it deleted? was it renamed?): if not Account.query.filter(Account.name == account_name).first(): app.logger.error("[X] Account has been removed or renamed: {}. Please restart the scheduler to fix.".format( account_name )) return try: reporter_logic(account_name, technology_name) time2 = time.time() app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name, technology_name, (time2 - time1))) app.logger.info( "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name)) except Exception as e: if sentry: sentry.captureException() app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e)) app.logger.error(traceback.format_exc()) store_exception("scheduler-exception-on-watch", None, e) raise self.retry(exc=e)
def task_audit(self, account_name, technology_name): setup() app.logger.info( "[ ] Executing Celery task to audit changes for Account: {} Technology: {}" .format(account_name, technology_name)) try: audit_changes([account_name], [technology_name], True) app.logger.info( "[+] Completed Celery task for account: {}, technology: {}".format( account_name, technology_name)) except Exception as e: if sentry: sentry.captureException() app.logger.error( "[X] Task Audit Scheduler Exception ({}/{}): {}".format( account_name, technology_name, e)) app.logger.error(traceback.format_exc()) store_exception("scheduler-exception-on-audit", None, e) self.retry(exc=e)
def decorated_function(*args, **kwargs): # prevent these from being passed to the wrapped function: m = kwargs.pop if pop_exception_fields else kwargs.get exception_values = { 'index': m('index'), 'account': m('account_name', None), 'exception_record_region': m('exception_record_region', None), 'name': m('name', None), 'exception_map': m('exception_map') } try: return f(*args, **kwargs) except Exception as e: if sentry: sentry.captureException() index = exception_values['index'] account = exception_values['account'] # Allow the recording region to be overridden for universal tech like IAM region = exception_values[ 'exception_record_region'] or kwargs.get('region') name = exception_values['name'] exception_map = exception_values['exception_map'] exc = BotoConnectionIssue(str(e), index, account, name) if name: location = (index, account, region, name) elif region: location = (index, account, region) elif account: location = (index, account) else: location = (index, ) exception_map[location] = exc # Store the exception (the original one passed in, not exc): store_exception(source=source, location=location, exception=e)
def setup_the_tasks(sender, **kwargs): setup() # Purge out all current tasks waiting to execute: purge_it() # Get the celery configuration (Get the raw module since Celery doesn't document a good way to do this # see https://github.com/celery/celery/issues/4633): celery_config = get_celery_config_file() # Add all the tasks: try: accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa for account in accounts: rep = Reporter(account=account.name) # Is this a dedicated watcher stack, or is this stack ignoring anything? only_watch = get_sm_celery_config_value(celery_config, "security_monkey_only_watch", set) # If only_watch is set, then ignoring is ignored. if only_watch: ignoring = set() else: # Check if we are ignoring any watchers: ignoring = get_sm_celery_config_value(celery_config, "security_monkey_watcher_ignore", set) or set() for monitor in rep.all_monitors: # Is this watcher enabled? if monitor.watcher.is_active() and monitor.watcher.index not in ignoring: # Did we specify specific watchers to run? if only_watch and monitor.watcher.index not in only_watch: continue app.logger.info("[ ] Scheduling tasks for {type} account: {name}".format(type=account.type.name, name=account.name)) interval = monitor.watcher.get_interval() if not interval: app.logger.debug("[/] Skipping watcher for technology: {} because it is set for external " "monitoring.".format(monitor.watcher.index)) continue app.logger.debug("[{}] Scheduling for technology: {}".format(account.type.name, monitor.watcher.index)) # Start the task immediately: task_account_tech.apply_async((account.name, monitor.watcher.index)) app.logger.debug("[-->] Scheduled immediate task") schedule = interval * 60 schedule_at_full_hour = get_sm_celery_config_value(celery_config, "schedule_at_full_hour", bool) or False if schedule_at_full_hour: if interval == 15: # 15 minute schedule = crontab(minute="0,15,30,45") elif interval == 60: # Hourly schedule = crontab(minute="0") elif interval == 720: # 12 hour schedule = crontab(minute="0", hour="0,12") elif interval == 1440: # Daily schedule = crontab(minute="0", hour="0") elif interval == 10080: # Weekly schedule = crontab(minute="0", hour="0", day_of_week="0") # Schedule it based on the schedule: sender.add_periodic_task(schedule, task_account_tech.s(account.name, monitor.watcher.index)) app.logger.debug("[+] Scheduled task to occur every {} minutes".format(interval)) # TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily # disabled this to avoid many duplicate events from getting added. # Also schedule a manual audit changer just in case it doesn't properly # audit (only for non-batched): # if not monitor.batch_support: # sender.add_periodic_task( # crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index)) # app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index)) # # app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name, # monitor.watcher.index)) app.logger.debug("[+] Completed scheduling tasks for account: {}".format(account.name)) # Schedule the task for clearing out old exceptions: app.logger.info("Scheduling task to clear out old exceptions.") # Run every 24 hours (and clear it now): clear_expired_exceptions.apply_async() sender.add_periodic_task(86400, clear_expired_exceptions.s()) except Exception as e: if sentry: sentry.captureException() app.logger.error("[X] Scheduler Exception: {}".format(e)) app.logger.error(traceback.format_exc()) store_exception("scheduler", None, e)
def setup_the_tasks(sender, **kwargs): setup() # Purge out all current tasks waiting to execute: purge_it() # Get the celery configuration (Get the raw module since Celery doesn't document a good way to do this # see https://github.com/celery/celery/issues/4633): celery_config = get_celery_config_file() # Add all the tasks: try: accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa for account in accounts: rep = Reporter(account=account.name) # Is this a dedicated watcher stack, or is this stack ignoring anything? only_watch = get_sm_celery_config_value(celery_config, "security_monkey_only_watch", set) # If only_watch is set, then ignoring is ignored. if only_watch: ignoring = set() else: # Check if we are ignoring any watchers: ignoring = get_sm_celery_config_value(celery_config, "security_monkey_watcher_ignore", set) or set() for monitor in rep.all_monitors: # Is this watcher enabled? if monitor.watcher.is_active() and monitor.watcher.index not in ignoring: # Did we specify specific watchers to run? if only_watch and monitor.watcher.index not in only_watch: continue app.logger.info("[ ] Scheduling tasks for {type} account: {name}".format(type=account.type.name, name=account.name)) interval = monitor.watcher.get_interval() * 60 if not interval: app.logger.debug("[/] Skipping watcher for technology: {} because it is set for external " "monitoring.".format(monitor.watcher.index)) continue app.logger.debug("[{}] Scheduling for technology: {}".format(account.type.name, monitor.watcher.index)) # Start the task immediately: task_account_tech.apply_async((account.name, monitor.watcher.index)) app.logger.debug("[-->] Scheduled immediate task") # Schedule it based on the schedule: sender.add_periodic_task(interval, task_account_tech.s(account.name, monitor.watcher.index)) app.logger.debug("[+] Scheduled task to occur every {} minutes".format(interval)) # TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily # disabled this to avoid many duplicate events from getting added. # Also schedule a manual audit changer just in case it doesn't properly # audit (only for non-batched): # if not monitor.batch_support: # sender.add_periodic_task( # crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index)) # app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index)) # # app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name, # monitor.watcher.index)) app.logger.debug("[+] Completed scheduling tasks for account: {}".format(account.name)) # Schedule the task for clearing out old exceptions: app.logger.info("Scheduling task to clear out old exceptions.") # Run every 24 hours (and clear it now): clear_expired_exceptions.apply_async() sender.add_periodic_task(86400, clear_expired_exceptions.s()) except Exception as e: if sentry: sentry.captureException() app.logger.error("[X] Scheduler Exception: {}".format(e)) app.logger.error(traceback.format_exc()) store_exception("scheduler", None, e)
def setup_the_tasks(sender, **kwargs): setup() # Purge out all current tasks waiting to execute: purge_it() # Add all the tasks: try: # TODO: Investigate options to have the scheduler skip different types of accounts accounts = Account.query.filter(Account.third_party == False).filter( Account.active == True).all() # noqa for account in accounts: app.logger.info( "[ ] Scheduling tasks for {type} account: {name}".format( type=account.type.name, name=account.name)) rep = Reporter(account=account.name) for monitor in rep.all_monitors: if monitor.watcher: app.logger.debug( "[{}] Scheduling for technology: {}".format( account.type.name, monitor.watcher.index)) interval = monitor.watcher.get_interval() * 60 # Start the task immediately: task_account_tech.apply_async( (account.name, monitor.watcher.index)) app.logger.debug("[-->] Scheduled immediate task") # Schedule it based on the schedule: sender.add_periodic_task( interval, task_account_tech.s(account.name, monitor.watcher.index)) app.logger.debug( "[+] Scheduled task to occur every {} minutes".format( interval)) # TODO: Due to a bug with Celery (https://github.com/celery/celery/issues/4041) we temporarily # disabled this to avoid many duplicate events from getting added. # Also schedule a manual audit changer just in case it doesn't properly # audit (only for non-batched): # if not monitor.batch_support: # sender.add_periodic_task( # crontab(hour=10, day_of_week="mon-fri"), task_audit.s(account.name, monitor.watcher.index)) # app.logger.debug("[+] Scheduled task for tech: {} for audit".format(monitor.watcher.index)) # # app.logger.debug("[{}] Completed scheduling for technology: {}".format(account.name, # monitor.watcher.index)) app.logger.debug( "[+] Completed scheduling tasks for account: {}".format( account.name)) # Schedule the task for clearing out old exceptions: app.logger.info("Scheduling task to clear out old exceptions.") # TODO: Investigate if this creates many duplicate tasks RE: Celery bug mentioned above sender.add_periodic_task(crontab(hour=3, minute=0), clear_expired_exceptions.s()) except Exception as e: if sentry: sentry.captureException() app.logger.error("[X] Scheduler Exception: {}".format(e)) app.logger.error(traceback.format_exc()) store_exception("scheduler", None, e)