def _upload(self, package_path):
"""Upload the StreamAlert package and sha256 sum to S3.
Args:
package path (str): Full path to the zipped dpeloyment package
Returns:
bool: Indicating a successful S3 upload
"""
LOGGER_CLI.info('Uploading StreamAlert package to S3')
client = boto3.client(
's3', region_name=self.config['global']['account']['region'])
for package_file in (package_path, '{}.sha256'.format(package_path)):
package_name = package_file.split('/')[-1]
package_fh = open(package_file, 'r')
try:
client.put_object(Bucket=self.config['lambda'][self.config_key]
['source_bucket'],
Key=os.path.join(self.package_name,
package_name),
Body=package_fh,
ServerSideEncryption='AES256')
except ClientError:
LOGGER_CLI.exception('An error occurred while uploading %s',
package_name)
return False
package_fh.close()
LOGGER_CLI.debug('Uploaded %s to S3', package_name)
return True
def terraform_generate(config, init=False):
"""Generate all Terraform plans for the configured clusters.
Keyword Args:
config (dict): The loaded config from the 'conf/' directory
init (bool): Indicates if main.tf.json is generated for `terraform init`
Returns:
bool: Result of cluster generating
"""
cleanup_old_tf_files(config)
# Setup the main.tf.json file
LOGGER_CLI.debug('Generating cluster file: main.tf.json')
with open('terraform/main.tf.json', 'w') as tf_file:
json.dump(generate_main(init=init, config=config),
tf_file,
indent=2,
sort_keys=True)
# Return early during the init process, clusters are not needed yet
if init:
return True
# Setup cluster files
for cluster in config.clusters():
if cluster in RESTRICTED_CLUSTER_NAMES:
raise InvalidClusterName(
'Rename cluster "main" or "athena" to something else!')
LOGGER_CLI.debug('Generating cluster file: %s.tf.json', cluster)
cluster_dict = generate_cluster(cluster_name=cluster, config=config)
if not cluster_dict:
LOGGER_CLI.error(
'An error was generated while creating the %s cluster',
cluster)
return False
with open('terraform/{}.tf.json'.format(cluster), 'w') as tf_file:
json.dump(cluster_dict, tf_file, indent=2, sort_keys=True)
# Setup Athena if it is enabled
athena_config = config['lambda'].get('athena_partition_refresh_config')
if athena_config:
athena_file = 'terraform/athena.tf.json'
if athena_config['enabled']:
athena_generated_config = generate_athena(config=config)
if athena_generated_config:
with open(athena_file, 'w') as tf_file:
json.dump(athena_generated_config,
tf_file,
indent=2,
sort_keys=True)
# Remove Athena file if it's disabled
else:
if os.path.isfile(athena_file):
LOGGER_CLI.info('Removing old Athena Terraform file')
os.remove(athena_file)
return True
def zip(temp_package_path):
"""Create the StreamAlert Lambda deployment package archive.
Zips all dependency files to run the function,
and names this zipfile based on the current date/time,
along with the Lambda function module version.
example filename: stream_alert_1.0.0_20161010_00:11:22.zip
Only package in the `.py` files per AWS's instructions
for creation of lambda functions.
Args:
temp_package_path (str): the temporary file path to store the zip.
Returns:
str: Deployment package full path
"""
LOGGER_CLI.debug('Creating Lambda package: %s',
temp_package_path + '.zip')
package_path = shutil.make_archive(temp_package_path, 'zip',
temp_package_path)
LOGGER_CLI.info('Package successfully created')
return package_path
def terraform_generate(config, init=False):
"""Generate all Terraform plans for the configured clusters.
Keyword Args:
config (dict): The loaded config from the 'conf/' directory
init (bool): Indicates if main.tf.json is generated for `terraform init`
Returns:
bool: Result of cluster generating
"""
cleanup_old_tf_files(config)
# Setup the main.tf.json file
LOGGER_CLI.debug('Generating cluster file: main.tf.json')
with open('terraform/main.tf.json', 'w') as tf_file:
json.dump(generate_main(init=init, config=config),
tf_file,
indent=2,
sort_keys=True)
# Return early during the init process, clusters are not needed yet
if init:
return True
# Setup cluster files
for cluster in config.clusters():
if cluster in RESTRICTED_CLUSTER_NAMES:
raise InvalidClusterName(
'Rename cluster "main" or "athena" to something else!')
LOGGER_CLI.debug('Generating cluster file: %s.tf.json', cluster)
cluster_dict = generate_cluster(cluster_name=cluster, config=config)
if not cluster_dict:
LOGGER_CLI.error(
'An error was generated while creating the %s cluster',
cluster)
return False
with open('terraform/{}.tf.json'.format(cluster), 'w') as tf_file:
json.dump(cluster_dict, tf_file, indent=2, sort_keys=True)
# Setup Athena if it is enabled
generate_global_lambda_settings(
config,
config_name='athena_partition_refresh_config',
config_generate_func=generate_athena,
tf_tmp_file='terraform/athena.tf.json',
message='Removing old Athena Terraform file')
# Setup Threat Intel Downloader Lambda function if it is enabled
generate_global_lambda_settings(
config,
config_name='threat_intel_downloader_config',
config_generate_func=generate_threat_intel_downloader,
tf_tmp_file='terraform/ti_downloader.tf.json',
message='Removing old Threat Intel Downloader Terraform file')
return True
def _cleanup(*files):
"""Removes the temporary StreamAlert package and checksum.
Args:
files (str): File paths to remove after uploading to S3.
"""
LOGGER_CLI.debug('Removing local files')
for obj in files:
os.remove(obj)
def tf_runner(**kwargs):
"""Terraform wrapper to build StreamAlert infrastructure.
Steps:
- resolve modules with `terraform get`
- run `terraform plan` for the given targets
- if plan is successful and user confirms prompt,
then the infrastructure is applied
kwargs:
targets: a list of Terraform targets
action: 'apply' or 'destroy'
Returns:
bool: True if the terraform command was successful
"""
targets = kwargs.get('targets', [])
action = kwargs.get('action', None)
tf_action_index = 1 # The index to the terraform 'action'
var_files = {'conf/lambda.json'}
tf_opts = ['-var-file=../{}'.format(x) for x in var_files]
tf_targets = ['-target={}'.format(x) for x in targets]
tf_command = ['terraform', 'plan'] + tf_opts + tf_targets
if action == 'destroy':
tf_command.append('-destroy')
LOGGER_CLI.debug('Resolving Terraform modules')
if not run_command(['terraform', 'get'], quiet=True):
return False
LOGGER_CLI.info('Planning infrastructure')
if not run_command(tf_command):
return False
if not continue_prompt():
sys.exit(1)
if action == 'destroy':
LOGGER_CLI.info('Destroying infrastructure')
tf_command[tf_action_index] = action
tf_command.remove('-destroy')
tf_command.append('-force')
elif action:
tf_command[tf_action_index] = action
else:
LOGGER_CLI.info('Creating infrastructure')
tf_command[tf_action_index] = 'apply'
tf_command.append('-refresh=false')
if not run_command(tf_command):
return False
return True
def _publish(self, client, function_name, code_sha_256):
"""Publish the function"""
date = datetime.utcnow().strftime("%Y%m%d_T%H%M%S")
LOGGER_CLI.debug('Publishing %s', function_name)
new_version = self._version_helper(client=client,
function_name=function_name,
code_sha_256=code_sha_256,
date=date)
return new_version
def _publish_helper(self, **kwargs):
"""Handle clustered or single Lambda function publishing
Keyword Arguments:
cluster [string]: The cluster to deploy to, this is optional
Returns:
[Boolean]: Result of the function publishes
"""
cluster = kwargs.get('cluster')
date = datetime.utcnow().strftime("%Y%m%d_T%H%M%S")
# Clustered Lambda functions have a different naming pattern
if cluster:
region = self.config['clusters'][cluster]['region']
function_name = '{}_{}_streamalert_{}'.format(
self.config['global']['account']['prefix'],
cluster,
self.package.package_name
)
else:
region = self.config['global']['account']['region']
function_name = '{}_streamalert_{}'.format(
self.config['global']['account']['prefix'],
self.package.package_name
)
# Configure the Lambda client
client = boto3.client('lambda', region_name=region)
code_sha_256 = self.config['lambda'][self.package.config_key]['source_current_hash']
# Publish the function
LOGGER_CLI.debug('Publishing %s', function_name)
new_version = self._version_helper(
client=client,
function_name=function_name,
code_sha_256=code_sha_256,
date=date)
if not new_version:
return False
# Update the config
if cluster:
LOGGER_CLI.info('Published version %s for %s:%s',
new_version, cluster, function_name)
self.config['clusters'][cluster]['modules']['stream_alert'][self.package.package_name]['current_version'] = new_version
else:
LOGGER_CLI.info('Published version %s for %s',
new_version, function_name)
self.config['lambda'][self.package.config_key]['current_version'] = new_version
self.config.write()
return True
def _publish(client, function_name, code_sha_256):
"""Publish the function"""
date = datetime.utcnow().strftime("%Y%m%d_T%H%M%S")
LOGGER_CLI.debug('Publishing %s', function_name)
try:
version = client.publish_version(
FunctionName=function_name,
CodeSha256=code_sha_256,
Description='Publish Lambda {} on {}'.format(
function_name, date))['Version']
except ClientError as err:
LOGGER_CLI.error(err)
return
return int(version)
def check_credentials():
"""Check for valid AWS credentials in environment variables
Returns:
bool: True any of the AWS env variables exist
"""
try:
response = boto3.client('sts').get_caller_identity()
except NoCredentialsError:
LOGGER_CLI.error('No valid AWS Credentials found in your environment!')
LOGGER_CLI.error('Please follow the setup instructions here: '
'https://www.streamalert.io/getting-started.html'
'#configure-aws-credentials')
return False
LOGGER_CLI.debug(
'Using credentials for user \'%s\' with user ID \'%s\' in account '
'\'%s\'', response['Arn'], response['UserId'], response['Account'])
return True
def tf_runner(action='apply', refresh=True, auto_approve=False, targets=None):
"""Terraform wrapper to build StreamAlert infrastructure.
Resolves modules with `terraform get` before continuing.
Args:
action (str): Terraform action ('apply' or 'destroy').
refresh (bool): If True, Terraform will refresh its state before applying the change.
auto_approve (bool): If True, Terraform will *not* prompt the user for approval.
targets (list): Optional list of affected targets.
If not specified, Terraform will run against all of its resources.
Returns:
bool: True if the terraform command was successful
"""
LOGGER_CLI.debug('Resolving Terraform modules')
if not run_command(['terraform', 'get'], quiet=True):
return False
tf_command = [
'terraform', action, '-var-file=../conf/lambda.json',
'-refresh={}'.format(str(refresh).lower())
]
if action == 'destroy':
# Terraform destroy has a '-force' flag instead of '-auto-approve'
LOGGER_CLI.info('Destroying infrastructure')
tf_command.append('-force={}'.format(str(auto_approve).lower()))
else:
LOGGER_CLI.info('%s changes',
'Applying' if auto_approve else 'Planning')
tf_command.append('-auto-approve={}'.format(str(auto_approve).lower()))
if targets:
tf_command.extend('-target={}'.format(x) for x in targets)
return run_command(tf_command)
