def cli_runner(options): """Main Stream Alert CLI handler Args: options (dict): command line arguments passed from the argparser. Contains the following keys for terraform commands: (command, subcommand, target) Contains the following keys for lambda commands: (command, subcommand, env, func, source) """ cli_load_message = ('Issues? Report here: ' 'https://github.com/airbnb/streamalert/issues') LOGGER_CLI.info(cli_load_message) if options.command == 'output': configure_output(options) elif options.command == 'lambda': lambda_runner(options) elif options.command == 'terraform': terraform_runner(options)
def run_command(runner_args, **kwargs): """Helper function to run commands with error handling. Args: runner_args (list): Commands to run via subprocess kwargs: cwd (str): A path to execute commands from error_message (str): Message to show if command fails quiet (bool): Whether to show command output or hide it """ default_error_message = "An error occurred while running: {}".format( ' '.join(runner_args) ) error_message = kwargs.get('error_message', default_error_message) default_cwd = 'terraform' cwd = kwargs.get('cwd', default_cwd) # Add the -force-copy flag for s3 state copying to suppress dialogs that # the user must type 'yes' into. if runner_args[0] == 'terraform': if runner_args[1] == 'init': runner_args.append('-force-copy') stdout_option = None if kwargs.get('quiet'): stdout_option = open(os.devnull, 'w') try: subprocess.check_call(runner_args, stdout=stdout_option, cwd=cwd) except subprocess.CalledProcessError as err: LOGGER_CLI.error('%s\n%s', error_message, err.cmd) return False except OSError as err: LOGGER_CLI.error('%s\n%s (%s)', error_message, err.strerror, runner_args[0]) return False return True
def generate_cloudwatch_metric_alarms(cluster_name, cluster_dict, config): """Add the CloudWatch Metric Alarms information to the Terraform cluster dict. Args: cluster_name (str): The name of the currently generating cluster cluster_dict (defaultdict): The dict containing all Terraform config for a given cluster. config (dict): The loaded config from the 'conf/' directory """ infrastructure_config = config['global'].get('infrastructure') if not (infrastructure_config and 'monitoring' in infrastructure_config): LOGGER_CLI.error( 'Invalid config: Make sure you declare global infrastructure options!' ) return sns_topic_arn = monitoring_topic_arn(config) cluster_dict['module']['stream_alert_{}'.format( cluster_name)]['sns_topic_arn'] = sns_topic_arn stream_alert_config = config['clusters'][cluster_name]['modules'][ 'stream_alert'] # Add cluster metric alarms for the rule and alert processors formatted_alarms = [] for func_config in stream_alert_config.values(): if 'metric_alarms' not in func_config: continue # TODO: update this logic to simply use a list of maps once Terraform fixes # their support for this, instead of the comma-separated string this creates metric_alarms = func_config['metric_alarms'] for name, alarm_info in metric_alarms.iteritems(): formatted_alarms.append(_format_metric_alarm(name, alarm_info)) cluster_dict['module']['stream_alert_{}'.format( cluster_name)]['metric_alarms'] = formatted_alarms
def format_record(test_record): """Create a properly formatted Kinesis, S3, or SNS record. Supports a dictionary or string based data record. Reads in event templates from the test/integration/templates folder. Args: test_record: Test record metadata dict with the following structure: data - string or dict of the raw data description - a string describing the test that is being performed trigger - bool of if the record should produce an alert source - which stream/s3 bucket originated the data service - which aws service originated the data Returns: dict in the format of the specific service """ service = test_record['service'] source = test_record['source'] data_type = type(test_record['data']) if data_type == dict: data = json.dumps(test_record['data']) elif data_type in (unicode, str): data = test_record['data'] else: LOGGER_CLI.info('Invalid data type: %s', type(test_record['data'])) return # Get the template file for this particular service template_path = os.path.join(DIR_TEMPLATES, '{}.json'.format(service)) with open(template_path, 'r') as service_template: try: template = json.load(service_template) except ValueError as err: LOGGER_CLI.error('Error loading %s.json: %s', service, err) return if service == 's3': # Set the S3 object key to a random value for testing test_record['key'] = ('{:032X}'.format(random.randrange(16**32))) template['s3']['object']['key'] = test_record['key'] template['s3']['bucket']['arn'] = 'arn:aws:s3:::{}'.format(source) template['s3']['bucket']['name'] = source # Create the mocked s3 object in the designated bucket with the random key put_mocked_s3_object(source, test_record['key'], data) elif service == 'kinesis': template['kinesis']['data'] = base64.b64encode(data) template[ 'eventSourceARN'] = 'arn:aws:kinesis:us-east-1:111222333:stream/{}'.format( source) elif service == 'sns': # TODO implement sns testing raise NotImplementedError else: LOGGER_CLI.info('Invalid service %s', service) return template
def create_and_upload(self): """Create a Lambda deployment package, hash it, and upload it to S3. Reference: package_name: Generated name based on date/time/version/name temp_package_path: Temp package to store deployment package files package_path: Full path to zipped deployment package package_sha256: Checksum of package_path package_sha256_path: Full path to package_path checksum file """ LOGGER_CLI.info('Creating package for %s', self.package_name) # get tmp dir and copy files temp_package_path = self._get_tmpdir() self._copy_files(temp_package_path) # download third-party libs if not self._resolve_third_party(temp_package_path): LOGGER_CLI.exception( 'Failed to install necessary third-party libraries') exit(1) # zip up files package_path = self.zip(temp_package_path) generated_package_name = package_path.split('/')[-1] # checksum files package_sha256, package_sha256_path = self._sha256sum(package_path) # upload to s3 if self._upload(package_path): # remove generated deployment files self._cleanup(package_path, package_sha256_path) # set new config values and update full_package_name = os.path.join(self.package_name, generated_package_name) # make all config changes here self.config['lambda'][ self.config_key]['source_object_key'] = full_package_name self.config['lambda'][ self.config_key]['source_current_hash'] = package_sha256 self.config.write()
def generate_s3_events(cluster_name, cluster_dict, config): """Add the S3 Events module to the Terraform cluster dict. Args: cluster_name (str): The name of the currently generating cluster cluster_dict (defaultdict): The dict containing all Terraform config for a given cluster. config (dict): The loaded config from the 'conf/' directory Returns: bool: Result of applying the s3_events module """ modules = config['clusters'][cluster_name]['modules'] s3_event_buckets = modules['s3_events'] # Detect legacy and convert if isinstance(s3_event_buckets, dict) and 's3_bucket_id' in s3_event_buckets: del config['clusters'][cluster_name]['modules']['s3_events'] s3_event_buckets = [{'bucket_id': s3_event_buckets['s3_bucket_id']}] config['clusters'][cluster_name]['modules']['s3_events'] = s3_event_buckets LOGGER_CLI.info('Converting legacy S3 Events config') config.write() for bucket_info in s3_event_buckets: if 'bucket_id' not in bucket_info: LOGGER_CLI.error('Config Error: Missing bucket_id key from s3_event configuration') return False cluster_dict['module']['s3_events_{}'.format(bucket_info['bucket_id'].replace( '.', '_'))] = { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${{module.stream_alert_{}.lambda_arn}}'.format(cluster_name), 'bucket_id': bucket_info['bucket_id'], 'enable_events': bucket_info.get('enable_events', True), 'lambda_role_id': '${{module.stream_alert_{}.lambda_role_id}}'.format(cluster_name) } return True
def terraform_generate(config, init=False): """Generate all Terraform plans for the configured clusters. Keyword Args: config (dict): The loaded config from the 'conf/' directory init (bool): Indicates if main.tf.json is generated for `terraform init` Returns: bool: Result of cluster generating """ cleanup_old_tf_files(config) # Setup the main.tf.json file LOGGER_CLI.debug('Generating cluster file: main.tf.json') with open('terraform/main.tf.json', 'w') as tf_file: json.dump(generate_main(init=init, config=config), tf_file, indent=2, sort_keys=True) # Return early during the init process, clusters are not needed yet if init: return True # Setup cluster files for cluster in config.clusters(): if cluster in RESTRICTED_CLUSTER_NAMES: raise InvalidClusterName( 'Rename cluster "main" or "athena" to something else!') LOGGER_CLI.debug('Generating cluster file: %s.tf.json', cluster) cluster_dict = generate_cluster(cluster_name=cluster, config=config) if not cluster_dict: LOGGER_CLI.error( 'An error was generated while creating the %s cluster', cluster) return False with open('terraform/{}.tf.json'.format(cluster), 'w') as tf_file: json.dump(cluster_dict, tf_file, indent=2, sort_keys=True) # Setup Athena if it is enabled generate_global_lambda_settings( config, config_name='athena_partition_refresh_config', config_generate_func=generate_athena, tf_tmp_file='terraform/athena.tf.json', message='Removing old Athena Terraform file') # Setup Threat Intel Downloader Lambda function if it is enabled generate_global_lambda_settings( config, config_name='threat_intel_downloader_config', config_generate_func=generate_threat_intel_downloader, tf_tmp_file='terraform/ti_downloader.tf.json', message='Removing old Threat Intel Downloader Terraform file') return True
def save_app_auth_info(app, info, overwrite=False): """Function to add app auth information to parameter store Args: info (dict): Required values needed to save the requested authentication information to AWS Parameter Store """ # Get all of the required authentication values from the user for this app integration auth_dict = {auth_key: user_input(info['description'], False, info['format']) for auth_key, info in app.required_auth_info().iteritems()} description = ('Required authentication information for the \'{}\' service for ' 'use in the \'{}\' app'.format(info['type'], info['app_name'])) # Save these to the parameter store param_name = '{}_{}'.format(info['function_name'], AppConfig.AUTH_CONFIG_SUFFIX) saved = save_parameter(info['region'], param_name, auth_dict, description, overwrite) if saved: LOGGER_CLI.info('App authentication info successfully saved to parameter store.') else: LOGGER_CLI.error('App authentication info was not saved to parameter store.') return saved
def generate_global_lambda_settings(config, config_name, generate_func, tf_tmp_file, message): """Generate settings for global Lambda functions Args: config (dict): lambda function settings read from 'conf/' directory config_name (str): keyname of lambda function settings in config. generate_func (func): method to generate lambda function settings. tf_tmp_file (str): filename of terraform file, generated by CLI. message (str): Message will be logged by LOGGER. """ if not config['lambda'].get(config_name): LOGGER_CLI.info('Config for \'%s\' not in lambda.json', config_name) remove_temp_terraform_file(tf_tmp_file, message) return if config['lambda'][config_name].get('enabled', True): generated_config = generate_func(config=config) if generated_config: with open(tf_tmp_file, 'w') as tf_file: json.dump(generated_config, tf_file, indent=2, sort_keys=True) else: remove_temp_terraform_file(tf_tmp_file, message)
def zip(temp_package_path): """Create the StreamAlert Lambda deployment package archive. Zips all dependency files to run the function, and names this zipfile based on the current date/time, along with the Lambda function module version. example filename: stream_alert_1.0.0_20161010_00:11:22.zip Only package in the `.py` files per AWS's instructions for creation of lambda functions. Args: temp_package_path (str): the temporary file path to store the zip. Returns: str: Deployment package full path """ LOGGER_CLI.debug('Creating Lambda package: %s', temp_package_path + '.zip') package_path = shutil.make_archive(temp_package_path, 'zip', temp_package_path) LOGGER_CLI.info('Package successfully created') return package_path
def toggle_metrics(self, enabled, clusters, lambda_functions): """Toggle CloudWatch metric logging and filter creation Args: enabled (bool): False if disabling metrics, true if enable_logging clusters (list): Clusters to enable or disable metrics on lambda_functions (list): Which lambda functions to enable or disable metrics on (rule, alert, or athena) """ for function in lambda_functions: if function == metrics.ATHENA_PARTITION_REFRESH_NAME: if 'athena_partition_refresh_config' in self.config['lambda']: self.config['lambda']['athena_partition_refresh_config'] \ ['enable_metrics'] = enabled else: LOGGER_CLI.error('No Athena configuration found; please initialize first.') continue for cluster in clusters: self.config['clusters'][cluster]['modules']['stream_alert'] \ [function]['enable_metrics'] = enabled self.write()
def generate_global_lambda_settings(config, **kwargs): """Generate settings of global Lambda funcitons, Athena and Threat Intel Downloader Args: config (dict): lambda function settings read from 'conf/' directory Keyword Args: config_name (str): keyname of lambda function settings in config. config_generate_func (func): method to generate lambda function settings. tf_tmp_file (str): filename of terraform file, generated by CLI. message (str): Message will be logged by LOGGER. """ config_name = kwargs.get('config_name') tf_tmp_file = kwargs.get('tf_tmp_file') if config_name and config['lambda'].get(config_name) and tf_tmp_file: if config['lambda'].get(config_name)['enabled']: generated_config = kwargs.get('config_generate_func')(config=config) if generated_config: with open(tf_tmp_file, 'w') as tf_file: json.dump(generated_config, tf_file, indent=2, sort_keys=True) else: if os.path.isfile(tf_tmp_file): LOGGER_CLI.info(kwargs.get('message')) os.remove(tf_tmp_file)
def send_creds_to_s3(region, bucket, key, blob_data): """Put the encrypted credential blob for this service and destination in s3 Args: region (str): AWS region to use for boto3 client bucket (str): The name of the s3 bucket to write the encrypted credentials to key (str): ID for the s3 object to write the encrypted credentials to blob_data (bytes): Cipher text blob from the kms encryption """ try: client = boto3.client('s3', region_name=region) client.put_object(Body=blob_data, Bucket=bucket, Key=key, ServerSideEncryption='AES256') return True except ClientError as err: LOGGER_CLI.error( 'An error occurred while sending credentials to S3 for key \'%s\' ' 'in bucket \'%s\': %s', key, bucket, err.response['Error']['Message']) return False
def user_input(requested_info, mask, input_restrictions): """Prompt user for requested information Args: requested_info (str): Description of the information needed mask (bool): Decides whether to mask input or not Returns: str: response provided by the user """ # pylint: disable=protected-access response = '' prompt = '\nPlease supply {}: '.format(requested_info) if not mask: while not response: response = raw_input(prompt) # Restrict having spaces or colons in items (applies to things like # descriptors, etc) if isinstance(input_restrictions, re._pattern_type): if not input_restrictions.match(response): LOGGER_CLI.error( 'The supplied input should match the following ' 'regular expression: %s', input_restrictions.pattern) return user_input(requested_info, mask, input_restrictions) else: if any(x in input_restrictions for x in response): LOGGER_CLI.error( 'The supplied input should not contain any of the following: %s', '"{}"'.format('", "'.join(input_restrictions))) return user_input(requested_info, mask, input_restrictions) else: while not response: response = getpass(prompt=prompt) return response
def terraform_generate(**kwargs): """Generate all Terraform plans for the configured clusters. Keyword Args: config [dict]: The loaded config from the 'conf/' directory init [bool]: Indicates if main.tf is generated for `terraform init` """ config = kwargs.get('config') init = kwargs.get('init', False) # Setup main LOGGER_CLI.info('Generating cluster file: main.tf') main_json = json.dumps(generate_main(init=init, config=config), indent=2, sort_keys=True) with open('terraform/main.tf', 'w') as tf_file: tf_file.write(main_json) # Break out early during the init process, clusters aren't needed yet if init: return True # Setup clusters for cluster in config.clusters(): if cluster == 'main': raise InvalidClusterName( 'Rename cluster "main" to something else!') LOGGER_CLI.info('Generating cluster file: %s.tf', cluster) cluster_json = json.dumps(generate_cluster(cluster_name=cluster, config=config), indent=2, sort_keys=True) with open('terraform/{}.tf'.format(cluster), 'w') as tf_file: tf_file.write(cluster_json) return True
def _resolve_third_party(self, temp_package_path): """Install all third-party packages into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: False if the pip command failed to install requirements, True otherwise """ third_party_libs = self.config['lambda'][self.config_key]['third_party_libraries'] # Return a default of True here if no libraries to install if not third_party_libs: LOGGER_CLI.info('No third-party libraries to install.') return True LOGGER_CLI.info( 'Installing third-party libraries: %s', ', '.join(third_party_libs)) pip_command = ['pip', 'install'] pip_command.extend(third_party_libs) pip_command.extend(['--upgrade', '--target', temp_package_path]) # Return True if the pip command is successfully run return run_command(pip_command, cwd=temp_package_path, quiet=True)
def _extract_precompiled_libs(self, temp_package_path): """Extract any precompiled third-party packages into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: True if precompiled libs were extracted successfully, False if some are missing """ dependency_files = { } # Map library name to location of its precompiled .zip file for path in self.package_files: if path.endswith('_dependencies.zip'): dependency_files[os.path.basename(path)] = path elif os.path.isdir(path): # Traverse directory looking for .zip files for root, _, package_files in os.walk(path): dependency_files.update({ package_file: os.path.join(root, package_file) for package_file in package_files if package_file.endswith('_dependencies.zip') }) for lib in self.precompiled_libs: libs_name = '_'.join([lib, 'dependencies.zip']) if libs_name not in dependency_files: LOGGER_CLI.error('Missing precompiled libs for package: %s', libs_name) return False # Copy the contents of the dependency zip to the package directory with zipfile.ZipFile(dependency_files[libs_name], 'r') as libs_file: libs_file.extractall(temp_package_path) return True
def _extract_precompiled_libs(self, temp_package_path): """Extract any precompiled third-party packages into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: False if the required libs were not found, True if otherwise """ # Return true immediately if there are no precompiled requirements for this package if not self.precompiled_libs: return True # Get any dependency files throughout the package folders that have # the _dependencies.zip suffix dependency_files = { package_file: os.path.join(root, package_file) for folder in self.package_folders for root, _, package_files in os.walk(folder) for package_file in package_files if package_file.endswith('_dependencies.zip') } for lib in self.precompiled_libs: libs_name = '_'.join([lib, 'dependencies.zip']) if libs_name not in dependency_files: LOGGER_CLI.error('Missing precompiled libs for package: %s', libs_name) return False # Copy the contents of the dependency zip to the package directory with zipfile.ZipFile(dependency_files[libs_name], 'r') as libs_file: libs_file.extractall(temp_package_path) return True
def generate_flow_logs(cluster_name, cluster_dict, config): """Add the VPC Flow Logs module to the Terraform cluster dict. Args: cluster_name [string]: The name of the currently generating cluster cluster_dict [defaultdict]: The dict containing all Terraform config for a given cluster. config [dict]: The loaded config from the 'conf/' directory Returns: [bool] Result of applying the flow_logs module """ modules = config['clusters'][cluster_name]['modules'] flow_log_group_name_default = '{}_{}_streamalert_flow_logs'.format( config['global']['account']['prefix'], cluster_name) flow_log_group_name = modules['flow_logs'].get( 'log_group_name', flow_log_group_name_default) if modules['flow_logs']['enabled']: cluster_dict['module']['flow_logs_{}'.format(cluster_name)] = { 'source': 'modules/tf_stream_alert_flow_logs', 'destination_stream_arn': '${{module.kinesis_{}.arn}}'.format(cluster_name), 'flow_log_group_name': flow_log_group_name } for flow_log_input in ('vpcs', 'subnets', 'enis'): input_data = modules['flow_logs'].get(flow_log_input) if input_data: cluster_dict['module']['flow_logs_{}'.format( cluster_name)][flow_log_input] = input_data return True else: LOGGER_CLI.info('Flow logs disabled, nothing to do') return False
def _terraform_clean(config): """Remove leftover Terraform statefiles and main/cluster files Args: config (CLIConfig): Loaded StreamAlert CLI """ LOGGER_CLI.info('Cleaning Terraform files') cleanup_files = [ '{}.tf.json'.format(cluster) for cluster in config.clusters() ] cleanup_files.extend([ 'athena.tf.json', 'main.tf.json', 'terraform.tfstate', 'terraform.tfstate.backup' ]) for tf_file in cleanup_files: file_to_remove = 'terraform/{}'.format(tf_file) if not os.path.isfile(file_to_remove): continue os.remove(file_to_remove) # Finally, delete the Terraform directory if os.path.isdir('terraform/.terraform/'): shutil.rmtree('terraform/.terraform/')
def set_prefix(self, prefix): """Set the Org Prefix in Global settings""" if not isinstance(prefix, (unicode, str)): LOGGER_CLI.error('Invalid prefix type, must be string') return self.config['global']['account']['prefix'] = prefix self.config['global']['terraform']['tfstate_bucket'] = self.config[ 'global']['terraform']['tfstate_bucket'].replace( 'PREFIX_GOES_HERE', prefix) self.config['lambda']['alert_processor_config'][ 'source_bucket'] = self.config['lambda']['alert_processor_config'][ 'source_bucket'].replace('PREFIX_GOES_HERE', prefix) self.config['lambda']['rule_processor_config'][ 'source_bucket'] = self.config['lambda']['rule_processor_config'][ 'source_bucket'].replace('PREFIX_GOES_HERE', prefix) self.config['lambda']['stream_alert_apps_config'][ 'source_bucket'] = self.config['lambda'][ 'stream_alert_apps_config']['source_bucket'].replace( 'PREFIX_GOES_HERE', prefix) self.write() LOGGER_CLI.info('Prefix successfully configured')
def _resolve_third_party(self, temp_package_path): """Install all third-party packages into the deployment package folder Args: temp_package_path [string]: Full path to temp package path Returns: [boolean] False if the pip command failed to install requirements, True otherwise """ third_party_libs = self.config['lambda'][ self.config_key]['third_party_libraries'] if third_party_libs: LOGGER_CLI.info('Installing third-party libraries: %s', ', '.join(third_party_libs)) pip_command = ['install'] pip_command.extend(third_party_libs) pip_command.extend(['--upgrade', '--target', temp_package_path]) # Return True if the pip result code is 0 return pip.main(pip_command) == 0 else: LOGGER_CLI.info('No third-party libraries to install.') # Return a default of True here if pip is not called return True
def _publish_helper(self, **kwargs): """Handle clustered or single Lambda function publishing Keyword Arguments: cluster [string]: The cluster to deploy to, this is optional Returns: [Boolean]: Result of the function publishes """ cluster = kwargs.get('cluster') date = datetime.utcnow().strftime("%Y%m%d_T%H%M%S") # Clustered Lambda functions have a different naming pattern if cluster: region = self.config['clusters'][cluster]['region'] function_name = '{}_{}_streamalert_{}'.format( self.config['global']['account']['prefix'], cluster, self.package.package_name ) else: region = self.config['global']['account']['region'] function_name = '{}_streamalert_{}'.format( self.config['global']['account']['prefix'], self.package.package_name ) # Configure the Lambda client client = boto3.client('lambda', region_name=region) code_sha_256 = self.config['lambda'][self.package.config_key]['source_current_hash'] # Publish the function LOGGER_CLI.debug('Publishing %s', function_name) new_version = self._version_helper( client=client, function_name=function_name, code_sha_256=code_sha_256, date=date) if not new_version: return False # Update the config if cluster: LOGGER_CLI.info('Published version %s for %s:%s', new_version, cluster, function_name) self.config['clusters'][cluster]['modules']['stream_alert'][self.package.package_name]['current_version'] = new_version else: LOGGER_CLI.info('Published version %s for %s', new_version, function_name) self.config['lambda'][self.package.config_key]['current_version'] = new_version self.config.write() return True
def user_input(requested_info, mask, input_restrictions): """Prompt user for requested information Args: requested_info (str): Description of the information needed mask (bool): Decides whether to mask input or not Returns: str: response provided by the user """ # pylint: disable=protected-access response = '' prompt = '\nPlease supply {}: '.format(requested_info) if not mask: while not response: response = raw_input(prompt) # Restrict having spaces or colons in items (applies to things like # descriptors, etc) valid_response = False if isinstance(input_restrictions, re._pattern_type): valid_response = input_restrictions.match(response) if not valid_response: LOGGER_CLI.error( 'The supplied input should match the following ' 'regular expression: %s', input_restrictions.pattern) elif callable(input_restrictions): # Functions can be passed here to perform complex validation of input # Transform the response with the validating function response = input_restrictions(response) valid_response = response is not None and response is not False if not valid_response: LOGGER_CLI.error( 'The supplied input failed to pass the validation ' 'function: %s', input_restrictions.__doc__) else: valid_response = not any(x in input_restrictions for x in response) if not valid_response: restrictions = ', '.join('\'{}\''.format(restriction) for restriction in input_restrictions) LOGGER_CLI.error( 'The supplied input should not contain any of the following: %s', restrictions) if not valid_response: return user_input(requested_info, mask, input_restrictions) else: while not response: response = getpass(prompt=prompt) return response
def _validate_options(options): if not options.interval: LOGGER_CLI.error('Missing command line argument --interval') return False if not options.timeout: LOGGER_CLI.error('Missing command line argument --timeout') return False if not options.memory: LOGGER_CLI.error('Missing command line argument --memory') return False return True
def create_and_upload(self): """Create a Lambda deployment package, hash it, and upload it to S3. Reference: package_name: Generated name based on date/time/version/name temp_package_path: Temp package to store deployment package files package_path: Full path to zipped deployment package package_sha256: Checksum of package_path package_sha256_path: Full path to package_path checksum file """ LOGGER_CLI.info('Creating package for %s', self.package_name) temp_package_path = self._get_tmpdir() self._copy_files(temp_package_path) if not self._resolve_third_party(temp_package_path): LOGGER_CLI.exception( 'Failed to install necessary third-party libraries') exit(1) # Extract any precompiled third-party libs for this package if not self._extract_precompiled_libs(temp_package_path): LOGGER_CLI.exception( 'Failed to extract precompiled third-party libraries') exit(1) # Zip up files package_path = self.zip(temp_package_path) generated_package_name = package_path.split('/')[-1] # SHA256 checksum files package_sha256, package_sha256_path = self._sha256sum(package_path) # Upload to s3 if not self._upload(package_path): return False self._cleanup(package_path, package_sha256_path) # Set new config values and update full_package_name = os.path.join(self.package_name, generated_package_name) self.config['lambda'][ self.config_key]['source_object_key'] = full_package_name self.config['lambda'][ self.config_key]['source_current_hash'] = package_sha256 self.config.write() return True
def create_database(athena_client): """Create the 'streamalert' Athena database Args: athena_client (boto3.client): Instantiated CLI AthenaClient """ if athena_client.check_database_exists(): LOGGER_CLI.info( 'The \'streamalert\' database already exists, nothing to do') return create_db_success, create_db_result = athena_client.run_athena_query( query='CREATE DATABASE streamalert') if create_db_success and create_db_result['ResultSet'].get('Rows'): LOGGER_CLI.info('streamalert database successfully created!') LOGGER_CLI.info('results: %s', create_db_result['ResultSet']['Rows'])
def check_credentials(): """Check for valid AWS credentials in environment variables Returns: bool: True any of the AWS env variables exist """ try: response = boto3.client('sts').get_caller_identity() except NoCredentialsError: LOGGER_CLI.error('No valid AWS Credentials found in your environment!') LOGGER_CLI.error('Please follow the setup instructions here: ' 'https://www.streamalert.io/getting-started.html' '#configure-aws-credentials') return False LOGGER_CLI.debug( 'Using credentials for user \'%s\' with user ID \'%s\' in account ' '\'%s\'', response['Arn'], response['UserId'], response['Account']) return True
def set_prefix(self, prefix): """Set the Org Prefix in Global settings""" if not isinstance(prefix, (unicode, str)): LOGGER_CLI.error('Invalid prefix type, must be string') return if '_' in prefix: LOGGER_CLI.error('Prefix cannot contain underscores') return tf_state_bucket = '{}.streamalert.terraform.state'.format(prefix) self.config['global']['account']['prefix'] = prefix self.config['global']['account'][ 'kms_key_alias'] = '{}_streamalert_secrets'.format(prefix) self.config['global']['terraform']['tfstate_bucket'] = tf_state_bucket self.config['lambda']['athena_partition_refresh_config'][ 'buckets'].clear() self.config['lambda']['athena_partition_refresh_config']['buckets'] \ ['{}.streamalerts'.format(prefix)] = 'alerts' self.write() LOGGER_CLI.info('Prefix successfully configured')
def tf_runner(action='apply', refresh=True, auto_approve=False, targets=None): """Terraform wrapper to build StreamAlert infrastructure. Resolves modules with `terraform get` before continuing. Args: action (str): Terraform action ('apply' or 'destroy'). refresh (bool): If True, Terraform will refresh its state before applying the change. auto_approve (bool): If True, Terraform will *not* prompt the user for approval. targets (list): Optional list of affected targets. If not specified, Terraform will run against all of its resources. Returns: bool: True if the terraform command was successful """ LOGGER_CLI.debug('Resolving Terraform modules') if not run_command(['terraform', 'get'], quiet=True): return False tf_command = [ 'terraform', action, '-var-file=../conf/lambda.json', '-refresh={}'.format(str(refresh).lower()) ] if action == 'destroy': # Terraform destroy has a '-force' flag instead of '-auto-approve' LOGGER_CLI.info('Destroying infrastructure') tf_command.append('-force={}'.format(str(auto_approve).lower())) else: LOGGER_CLI.info('%s changes', 'Applying' if auto_approve else 'Planning') tf_command.append('-auto-approve={}'.format(str(auto_approve).lower())) if targets: tf_command.extend('-target={}'.format(x) for x in targets) return run_command(tf_command)