def _determine_last_time(self): """Determine the last time this function was executed and fallback on evaluating the rate value if there is no last timestamp available Returns: int: The unix timestamp for the starting point to fetch logs back to """ if not self.last_timestamp: interval_time = self.evaluate_interval() current_time = int(calendar.timegm(time.gmtime())) time_delta = current_time - interval_time LOGGER.debug( 'Current timestamp: %s seconds. Calculated delta: %s seconds', current_time, time_delta) # Request the date format from the app since some services expect different types # Using init=False will return the class without instantiating it date_format = StreamAlertApp.get_app(self, init=False).date_formatter() if date_format: self.last_timestamp = datetime.utcfromtimestamp( time_delta).strftime(date_format) else: self.last_timestamp = time_delta LOGGER.info('Starting last timestamp set to: %s', self.last_timestamp) return self.last_timestamp
def handler(event, context): """Main lambda handler use as the entry point Args: event (dict): Event object that can potentially contain details on what to during this invocation. An example of this is the 'invocation_type' key that is used as an override to allow for successive invocations (and in the future, support for historical invocations) context (LambdaContxt): AWS LambdaContext object """ try: # Load the config from this context object, pulling info from parameter store # The event object can contain detail about what to do, ie: 'invocation_type' config = AppConfig.load_config(context, event) # The config specifies what app this function is supposed to run app = StreamAlertApp.get_app(config) # Run the gather operation app.gather() finally: # If the config was loaded, save a bad state if the current state is still # marked as 'running' (aka not 'success' or 'partial' runs) if 'config' in locals() and config.is_running: config.mark_failure()
def add_app_integration(self, app_info): """Add a configuration for a new streamalert app integration function Args: app_info (dict): The necessary values needed to begin configuring a new app integration """ exists, prompt_for_auth, overwrite = False, True, False app = StreamAlertApp.get_app(app_info, False) # Check to see if there is an existing configuration for this app integration cluster_config = self.config['clusters'][app_info['cluster']] if app_info['app_name'] in cluster_config['modules'].get( 'stream_alert_apps', {}): prompt = ( 'An app with the name \'{}\' is already configured for cluster ' '\'{}\'. Would you like to update the existing app\'s configuration' '?'.format(app_info['app_name'], app_info['cluster'])) exists = True # Return if the user is not deliberately updating an existing config if not continue_prompt(message=prompt): return prompt = ( 'Would you also like to update the authentication information for ' 'app integration with name \'{}\'?'.format( app_info['app_name'])) # If this is true, we shouldn't prompt again to warn about overwriting prompt_for_auth = overwrite = continue_prompt(message=prompt) if prompt_for_auth and not save_app_auth_info(app, app_info, overwrite): return apps_config = cluster_config['modules'].get('stream_alert_apps', {}) local_config_keys = {'interval', 'timeout', 'memory'} if not exists: # Save a default log level as info to the config app_info['log_level'] = 'info' app_info['current_version'] = '$LATEST' local_config_keys.update({'log_level', 'current_version', 'type'}) apps_config[app_info['app_name']] = { key: app_info[key] for key in local_config_keys } else: apps_config[app_info['app_name']].update( {key: app_info[key] for key in local_config_keys}) cluster_config['modules']['stream_alert_apps'] = apps_config # Add this service to the sources for this app integration # The `stream_alert_app` is purposely singular here app_sources = self.config['sources'].get('stream_alert_app', {}) app_sources[app_info['function_name']] = {'logs': [app.service()]} self.config['sources']['stream_alert_app'] = app_sources LOGGER_CLI.info( 'Successfully added \'%s\' app integration to \'conf/clusters/%s.json\' ' 'for service \'%s\'.', app_info['app_name'], app_info['cluster'], app_info['type']) self.write()
def add_app_integration(self, app_info): """Add a configuration for a new streamalert app integration function Args: app_info (dict): The necessary values needed to begin configuring a new app integration """ exists, prompt_for_auth, overwrite = False, True, False app = StreamAlertApp.get_app(app_info, False) cluster_name = app_info['cluster'] app_name = app_info['app_name'] func_name = app_info['function_name'] # Check to see if there is an existing configuration for this app integration cluster_config = self.config['clusters'][cluster_name] if func_name in cluster_config['modules'].get('stream_alert_apps', {}): prompt = ( 'An app with the name \'{}\' is already configured for cluster ' '\'{}\'. Would you like to update the existing app\'s configuration' '?'.format(app_name, cluster_name)) exists = True # Return if the user is not deliberately updating an existing config if not continue_prompt(message=prompt): return prompt = ( 'Would you also like to update the authentication information for ' 'app integration with name \'{}\'?'.format(app_name)) # If this is true, we shouldn't prompt again to warn about overwriting prompt_for_auth = overwrite = continue_prompt(message=prompt) if prompt_for_auth and not save_app_auth_info(app, app_info, overwrite): return apps_config = cluster_config['modules'].get('stream_alert_apps', {}) if not exists: # Save a default app settings to the config for new apps new_app_config = { 'app_name': app_info['app_name'], 'concurrency_limit': 2, 'log_level': 'info', 'log_retention_days': 14, 'memory': app_info['memory'], 'metric_alarms': { 'errors': { 'enabled': True, 'evaluation_periods': 1, 'period_secs': 120 } }, 'schedule_expression': app_info['schedule_expression'], 'timeout': app_info['timeout'], 'type': app_info['type'] } apps_config[func_name] = new_app_config else: # Allow for updating certain attributes for the app without overwriting # current parts of the configuration updated_app_config = { 'memory': app_info['memory'], 'schedule_expression': app_info['schedule_expression'], 'timeout': app_info['timeout'] } apps_config[func_name].update(updated_app_config) cluster_config['modules']['stream_alert_apps'] = apps_config # Add this service to the sources for this app integration # The `stream_alert_app` is purposely singular here app_sources = self.config['sources'].get('stream_alert_app', {}) app_sources[app_info['function_name']] = {'logs': [app.service()]} self.config['sources']['stream_alert_app'] = app_sources LOGGER_CLI.info( 'Successfully added \'%s\' app integration to \'conf/clusters/%s.json\' ' 'for service \'%s\'.', app_info['app_name'], app_info['cluster'], app_info['type']) self.write()
def test_get_app_exception_invalid(): """App Integration - App Base, Get App Exception for Invalid Service""" config = AppConfig(get_valid_config_dict('duo_auth')) config['type'] = 'bad_service_type' StreamAlertApp.get_app(config)
def test_get_app_exception_type(): """App Integration - App Base, Get App Exception for No 'type'""" config = AppConfig(get_valid_config_dict('duo_auth')) del config['type'] StreamAlertApp.get_app(config)
def test_get_app(): """App Integration - App Base, Get App""" config = AppConfig(get_valid_config_dict('duo_auth')) app = StreamAlertApp.get_app(config) assert_is_not_none(app)
def _app_integration_handler(options): """Perform app integration related functions Args: options (argparser): Contains all of the necessary info for configuring a new app integration or updating an existing one """ if not options: return # Convert the options to a dict app_info = vars(options) # Add the region and prefix for this StreamAlert instance to the app info app_info['region'] = str(CONFIG['global']['account']['region']) app_info['prefix'] = str(CONFIG['global']['account']['prefix']) # Function name follows the format: '<prefix>_<cluster>_<service>_<app_name>_app func_parts = ['prefix', 'cluster', 'type', 'app_name'] # Create a new app integration function if options.subcommand == 'new': app_info['function_name'] = '_'.join( [app_info.get(value) for value in func_parts] + ['app']) CONFIG.add_app_integration(app_info) return # Update the auth information for an existing app integration function if options.subcommand == 'update-auth': cluster_config = CONFIG['clusters'][app_info['cluster']] if not app_info['app_name'] in cluster_config['modules'].get( 'stream_alert_apps', {}): LOGGER_CLI.error( 'App integration with name \'%s\' does not exist for cluster \'%s\'', app_info['app_name'], app_info['cluster']) return # Get the type for this app integration from the current # config so we can update it properly app_info['type'] = cluster_config['modules']['stream_alert_apps'] \ [app_info['app_name']]['type'] app_info['function_name'] = '_'.join( [app_info.get(value) for value in func_parts] + ['app']) app = StreamAlertApp.get_app(app_info, False) if not save_app_auth_info(app, app_info, True): return return # List all of the available app integrations, broken down by cluster if options.subcommand == 'list': all_info = { cluster: cluster_config['modules'].get('stream_alert_apps') for cluster, cluster_config in CONFIG['clusters'].iteritems() } for cluster, info in all_info.iteritems(): print '\nCluster: {}\n'.format(cluster) if not info: print '\tNo Apps configured\n' continue for name, details in info.iteritems(): print '\tName: {}'.format(name) print '\n'.join([ '\t\t{key}:{padding_char:<{padding_count}}{value}'.format( key=key_name, padding_char=' ', padding_count=30 - (len(key_name)), value=value) for key_name, value in details.iteritems() ] + ['\n'])