def load_bosslets():
n = lambda c: normalize(c).replace('_', '.')
bosslets = [n(c) for c in glob(repo_path('config', '*.py'))]
custom = [n(c) for c in glob(repo_path('config', 'custom', '*.py'))]
bosslets.extend(custom)
return bosslets
def load_lambdas_on_s3(session, domain, bucket):
"""Zip up spdb, bossutils, lambda and lambda_utils. Upload to S3.
Uses the lambda build server (an Amazon Linux AMI) to compile C code and
prepare the virtualenv that's ultimately contained in the zip file placed
in S3.
Args:
session (Session): boto3.Session
domain (string): The VPC's domain name such as integration.boss.
"""
tempname = tempfile.NamedTemporaryFile(delete=True)
zipname = tempname.name + '.zip'
tempname.close()
print('Using temp zip file: ' + zipname)
cwd = os.getcwd()
os.chdir(const.repo_path("salt_stack", "salt", "spdb", "files"))
zip.write_to_zip('spdb.git', zipname, False)
os.chdir(cwd)
os.chdir(const.repo_path("salt_stack", "salt", "boss-tools", "files", "boss-tools.git"))
zip.write_to_zip('bossutils', zipname)
zip.write_to_zip('lambda', zipname)
zip.write_to_zip('lambdautils', zipname)
os.chdir(cwd)
with open(NDINGEST_SETTINGS_TEMPLATE, 'r') as tmpl:
# Generate settings.ini file for ndingest.
create_ndingest_settings(domain, tmpl)
os.chdir(const.repo_path("salt_stack", "salt", "ndingest", "files"))
zip.write_to_zip('ndingest.git', zipname)
os.chdir(cwd)
print("Copying local modules to lambda-build-server")
#copy the zip file to lambda_build_server
lambda_build_server = aws.get_lambda_server(session)
lambda_build_server_key = aws.get_lambda_server_key(session)
lambda_build_server_key = utils.keypair_to_file(lambda_build_server_key)
ssh = SSHConnection(lambda_build_server_key, (lambda_build_server, 22, 'ec2-user'))
target_file = "sitezips/{}.zip".format(domain)
ret = ssh.scp(zipname, target_file, upload=True)
print("scp return code: " + str(ret))
os.remove(zipname)
# This section will run makedomainenv on lambda-build-server
print("calling makedomainenv on lambda-build-server")
cmd = 'source /etc/profile && source ~/.bash_profile && /home/ec2-user/makedomainenv {} {}'.format(domain, bucket)
ssh.cmd(cmd)
def update_migrate(bosslet_config, config):
migration_progress = constants.repo_path("cloud_formation", "configs", "migrations", config, "progress")
if not os.path.exists(migration_progress):
console.info("No migrations to apply")
return
with open(migration_progress, "r") as fh:
cur_ver = int(fh.read())
new_ver = CloudFormationConfiguration(config, bosslet_config).existing_version()
migrations = MigrationManager(config, cur_ver, new_ver)
if not migrations.has_migrations:
console.info("No migrations to apply")
os.remove(migration_progress)
return
def callback(migration_file):
with open(migration_progress, 'w') as fh:
fh.write(str(migration_file.stop))
migrations.add_callback(post=callback)
migrations.post_update(bosslet_config)
os.remove(migration_progress)
def load_scenarios():
scenarios = [
normalize(s)
for s in glob(repo_path('cloud_formation', 'scenarios', '*.yml'))
]
return scenarios
def load_cf_configs():
configs = [
normalize(c)
for c in glob(repo_path('cloud_formation', 'configs', '*.py'))
]
configs.append('all')
configs.remove('__init__')
return configs
def load_lambda_config(lambda_dir):
"""Load the lambda.yml config file
Args:
lambda_dir (str): Name of the directory under cloud_formation/lambda/ that
contains the lambda.yml file to load
Returns:
dict: Dictionary of configuration file data
"""
lambda_config = const.repo_path('cloud_formation', 'lambda', lambda_dir, 'lambda.yml')
with open(lambda_config, 'r') as fh:
return yaml.full_load(fh.read())
def pre_init(bosslet_config):
"""
Create NodeJS config file from template.
Package NodeJS lambda function.
Upload .zip to S3 bucket.
"""
write_config_file(bosslet_config)
copy_provisioners(bosslet_config)
# Inject the current region into the lambda code
region_json = const.repo_path('cloud_formation', 'lambda',
'dynamodb-lambda-autoscale', 'src',
'configuration', 'Region.json')
with open(region_json, 'w') as fh:
json.dump({'Region': bosslet_config.REGION}, fh)
lambdas.load_lambdas_on_s3(
bosslet_config,
bosslet_config.names.dynamo_lambda.lambda_,
)
import os
import pprint
import datetime
import json
from boto3 import Session
from botocore.exceptions import ClientError
import alter_path
from lib import aws
from lib import utils
from lib import hosts
from lib import constants as const
from lib.boto_wrapper import IamWrapper as iw
IAM_CONFIG_DIR = const.repo_path("config", "iam")
DEFAULT_POLICY_FILE = os.path.join(IAM_CONFIG_DIR, "policies.json")
DEFAULT_GROUP_FILE = os.path.join(IAM_CONFIG_DIR, "groups.json")
DEFAULT_ROLES_FILE = os.path.join(IAM_CONFIG_DIR, "roles.json")
COMMANDS=["import", "export"]
class IamUtils:
def __init__(self, session):
self.session = session
self.iam_details = None
self.iw = iw(session.client("iam"))
self.policy_keyword_filters = ["-client-policy-"] # Any keywords in the policy name should be skipped.
self.policy_whole_filters = ["gion-test-policy", "aplAllowAssumeRoleInProduction",
"aplDenyAssumeRoleInProduction", "aplSpeedTestPolicy", "drenkng1-agents-rw-policy"]
self.role_keyword_filters = ["cowlehp1"]
self.role_whole_filters = ["drenkng1-agents-rw-policy", "alexaSkillsRole", "ecrReadOnlyRole", "ecsInstanceRole",
def load_lambdas_on_s3(session, domain, bucket):
"""Zip up spdb, bossutils, lambda and lambda_utils. Upload to S3.
Uses the lambda build server (an Amazon Linux AMI) to compile C code and
prepare the virtualenv that's ultimately contained in the zip file placed
in S3.
Args:
session (Session): boto3.Session
domain (string): The VPC's domain name such as integration.boss.
"""
tempname = tempfile.NamedTemporaryFile(delete=True)
zipname = tempname.name + '.zip'
tempname.close()
print('Using temp zip file: ' + zipname)
cwd = os.getcwd()
os.chdir(const.repo_path("salt_stack", "salt", "spdb", "files"))
zip.write_to_zip('spdb.git', zipname, False)
os.chdir(cwd)
os.chdir(const.repo_path("salt_stack", "salt", "boss-tools", "files", "boss-tools.git"))
zip.write_to_zip('bossutils', zipname)
zip.write_to_zip('lambda', zipname)
zip.write_to_zip('lambdautils', zipname)
os.chdir(cwd)
with open(NDINGEST_SETTINGS_TEMPLATE, 'r') as tmpl:
# Generate settings.ini file for ndingest.
create_ndingest_settings(domain, tmpl)
os.chdir(const.repo_path("salt_stack", "salt", "ndingest", "files"))
zip.write_to_zip('ndingest.git', zipname)
os.chdir(cwd)
os.chdir(const.repo_path("lib"))
zip.write_to_zip('heaviside.git', zipname)
# Let lambdas look up names by creating a bossnames module.
zip.write_to_zip('names.py', zipname, arcname='bossnames/names.py')
zip.write_to_zip('hosts.py', zipname, arcname='bossnames/hosts.py')
zip.write_to_zip('bucket_object_tags.py', zipname, arcname='bossnames/bucket_object_tags.py')
zip.write_to_zip('__init__.py', zipname, arcname='bossnames/__init__.py')
os.chdir(cwd)
print("Copying local modules to lambda-build-server")
#copy the zip file to lambda_build_server
lambda_build_server = aws.get_lambda_server(session)
lambda_build_server_key = aws.get_lambda_server_key(session)
lambda_build_server_key = utils.keypair_to_file(lambda_build_server_key)
ssh = SSHConnection(lambda_build_server_key, (lambda_build_server, 22, 'ec2-user'))
target_file = "sitezips/{}.zip".format(domain)
ret = ssh.scp(zipname, target_file, upload=True)
print("scp return code: " + str(ret))
os.remove(zipname)
# This section will run makedomainenv on lambda-build-server
print("calling makedomainenv on lambda-build-server")
cmd = 'source /etc/profile && source ~/.bash_profile && /home/ec2-user/makedomainenv {} {}'.format(domain, bucket)
ssh.cmd(cmd)
# This was an attempt to import CUBOIDSIZE from the spdb repo. Can't import
# without a compiling spdb's C library.
#
#SPDB_FOLDER = '../salt_stack/salt/spdb/files'
#SPDB_REPO = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER + '/spdb.git'))
#SPDB_LINK = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER + '/spdb'))
# try:
# os.symlink(SPDB_REPO, SPDB_LINK, True)
# spdb_dir = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER))
# sys.path.append(spdb_dir)
# from spdb.c_lib.c_version.ndtype import CUBOIDSIZE
# finally:
# os.remove(SPDB_LINK)
# Location of settings files for ndingest.
NDINGEST_SETTINGS_FOLDER = const.repo_path('salt_stack', 'salt', 'ndingest', 'files', 'ndingest.git', 'settings')
# Template used for ndingest settings.ini generation.
NDINGEST_SETTINGS_TEMPLATE = NDINGEST_SETTINGS_FOLDER + '/settings.ini.apl'
def get_lambda_zip_name(domain):
"""Get name of zip file containing lambda.
This must match the name created in the makedomainenv script that runs on
the lambda build server.
Args:
domain (string): The VPC's domain name such as integration.boss.
Returns:
(string)
"""
import argparse
import sys
import os
import glob
import json
import shlex
import subprocess
from distutils.spawn import find_executable
from boto3.session import Session
import alter_path
from lib.constants import repo_path
os.environ["PATH"] += ":" + repo_path("bin") # allow executing Packer from the bin/ directory
def get_commit():
"""Figure out the commit hash of the current git revision.
Note: Only works if the CWD is a git repository
Returns:
(string) : The git commit hash
"""
cmd = "git rev-parse HEAD"
result = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE)
return result.stdout.decode("utf-8").strip()
def execute(cmd, output_file):
"""Execuit the given command and redirect STDOUT and STDERR to output_file.
Args:
cmd (string) : Command to execute
# This was an attempt to import CUBOIDSIZE from the spdb repo. Can't import
# without a compiling spdb's C library.
#
#SPDB_FOLDER = '../salt_stack/salt/spdb/files'
#SPDB_REPO = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER + '/spdb.git'))
#SPDB_LINK = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER + '/spdb'))
# try:
# os.symlink(SPDB_REPO, SPDB_LINK, True)
# spdb_dir = os.path.normpath(os.path.join(cur_dir, SPDB_FOLDER))
# sys.path.append(spdb_dir)
# from spdb.c_lib.c_version.ndtype import CUBOIDSIZE
# finally:
# os.remove(SPDB_LINK)
# Location of settings files for ndingest.
NDINGEST_SETTINGS_FOLDER = const.repo_path('salt_stack', 'salt', 'ndingest', 'files', 'ndingest.git', 'settings')
# Template used for ndingest settings.ini generation.
NDINGEST_SETTINGS_TEMPLATE = NDINGEST_SETTINGS_FOLDER + '/settings.ini.apl'
def get_lambda_zip_name(domain):
"""Get name of zip file containing lambda.
This must match the name created in the makedomainenv script that runs on
the lambda build server.
Args:
domain (string): The VPC's domain name such as integration.boss.
Returns:
(string)
from boto3 import Session
from botocore.exceptions import ClientError
import alter_path
from lib import aws
from lib import utils
from lib import constants as const
from lib.boto_wrapper import IamWrapper
from lib import configuration
from lib import console
REGION_STANDIN = '==region=='
ACCOUNT_STANDIN = '==account=='
IAM_CONFIG_DIR = const.repo_path("config", "iam")
DEFAULT_POLICY_FILE = os.path.join(IAM_CONFIG_DIR, "policies.json")
DEFAULT_GROUP_FILE = os.path.join(IAM_CONFIG_DIR, "groups.json")
DEFAULT_ROLES_FILE = os.path.join(IAM_CONFIG_DIR, "roles.json")
DEFAULT_NAMES_FILE = os.path.join(IAM_CONFIG_DIR, "names.json")
DEFAULT_REMOVED_FILE = os.path.join(IAM_CONFIG_DIR, "removed.json")
class AllResources(object):
"""Class that always returns True for any `obj in AllResources()`"""
def __contains__(self, value):
return True
def json_dumps(obj):
"""A wrapper to `json.dumps` that provides the common arguments used
"""
import argparse
import sys
import os
import glob
import json
import shlex
import subprocess
from distutils.spawn import find_executable
from boto3.session import Session
import alter_path
from lib.constants import repo_path
os.environ["PATH"] += ":" + repo_path("bin") # allow executing Packer from the bin/ directory
def get_commit():
"""Figure out the commit hash of the current git revision.
Note: Only works if the CWD is a git repository
Returns:
(string) : The git commit hash
"""
cmd = "git rev-parse HEAD"
result = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE)
return result.stdout.decode("utf-8").strip()
def execute(cmd, output_file):
"""Execuit the given command and redirect STDOUT and STDERR to output_file.
Args:
cmd (string) : Command to execute
Starting ASG that already is on -
Initializing Vault -
Unsealing Vault -
Exported Vault data not existing -
Importing Vault data -
Stopping:
An error when stopping ASG that is on -
Stopping ASG that is already off -
Exporting Vault data -
Stopping Auth without RDS -
"""
KEY_PREFIX = 'boss_switch.'
VAULT_FILE = constants.repo_path('vault', 'private', '{}', 'export.json')
def load_aws(bosslet_config, method):
suffix = '.' + bosslet_config.INTERNAL_DOMAIN
client = bosslet_config.session.client('autoscaling')
response = client.describe_auto_scaling_groups()
def name(tags):
for tag in tags:
if tag['Key'] == 'Name':
return tag['Value']
return ""
asgs = [
def load_lambdas_on_s3(bosslet_config, lambda_name = None, lambda_dir = None):
"""Package up the lambda files and send them through the lambda build process
where the lambda code zip is produced and uploaded to S3
NOTE: This function is also used to build lambda layer code zips, the only requirement
for a layer is that the files in the resulting zip should be in the correct
subdirectory (`python/` for Python libraries) so that when a lambda uses the
layer the libraries included in the layer can be correctly loaded
NOTE: If lambda_name and lambda_dir are both None then lambda_dir is set to
'multi_lambda' for backwards compatibility
Args:
bosslet_config (BossConfiguration): Configuration object of the stack the
lambda will be deployed into
lambda_name (str): Name of the lambda, which will be mapped to the name of the
lambda directory that contains the lambda's code
lambda_dir (str): Name of the directory in `cloud_formation/lambda/` that
contains the `lambda.yml` configuration file for the lambda
Raises:
BossManageError: If there was a problem with building the lambda code zip or
uploading it to the given S3 bucket
"""
# For backwards compatibility build the multi_lambda code zip
if lambda_name is None and lambda_dir is None:
lambda_dir = 'multi_lambda'
# Map from lambda_name to lambda_dir if needed
if lambda_dir is None:
try:
lambda_dir = lambda_dirs(bosslet_config)[lambda_name]
except KeyError:
console.error("Cannot build a lambda that doesn't use a code zip file")
return None
# To prevent rubuilding a lambda code zip multiple times during an individual execution memorize what has been built
if lambda_dir in BUILT_ZIPS:
console.debug('Lambda code {} has already be build recently, skipping...'.format(lambda_dir))
return
BUILT_ZIPS.append(lambda_dir)
lambda_dir = pathlib.Path(const.repo_path('cloud_formation', 'lambda', lambda_dir))
lambda_config = lambda_dir / 'lambda.yml'
with lambda_config.open() as fh:
lambda_config = yaml.full_load(fh.read())
if lambda_config.get('layers'):
for layer in lambda_config['layers']:
# Layer names should end with `layer`
if not layer.endswith('layer'):
console.warning("Layer '{}' doesn't conform to naming conventions".format(layer))
load_lambdas_on_s3(bosslet_config, lambda_dir=layer)
console.debug("Building {} lambda code zip".format(lambda_dir))
domain = bosslet_config.INTERNAL_DOMAIN
tempname = tempfile.NamedTemporaryFile(delete=True)
zipname = pathlib.Path(tempname.name + '.zip')
tempname.close()
console.debug('Using temp zip file: {}'.format(zipname))
cwd = os.getcwd()
# Copy the lambda files into the zip
for filename in lambda_dir.glob('*'):
zip.write_to_zip(str(filename), zipname, arcname=filename.name)
# Copy the other files that should be included
if lambda_config.get('include'):
for src in lambda_config['include']:
dst = lambda_config['include'][src]
src_path, src_file = src.rsplit('/', 1)
os.chdir(const.repo_path(src_path))
# Generate dynamic configuration files, as needed
if src_file == 'ndingest.git':
with open(NDINGEST_SETTINGS_TEMPLATE, 'r') as tmpl:
# Generate settings.ini file for ndingest.
create_ndingest_settings(bosslet_config, tmpl)
zip.write_to_zip(src_file, zipname, arcname=dst)
os.chdir(cwd)
# Currently any Docker CLI compatible container setup can be used (like podman)
CONTAINER_CMD = '{EXECUTABLE} run --rm -it --env AWS_* --volume {HOST_DIR}:/var/task/ lambci/lambda:build-{RUNTIME} {CMD}'
BUILD_CMD = 'python3 {PREFIX}/build_lambda.py {DOMAIN} {BUCKET}'
BUILD_ARGS = {
'DOMAIN': domain,
'BUCKET': bosslet_config.LAMBDA_BUCKET,
}
# DP NOTE: not sure if this should be in the bosslet_config, as it is more about the local dev
# environment instead of the stack's environment. Different maintainer may have different
# container commands installed.
container_executable = os.environ.get('LAMBDA_BUILD_CONTAINER')
lambda_build_server = bosslet_config.LAMBDA_SERVER
if lambda_build_server is None:
staging_target = pathlib.Path(const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files', 'staging'))
if not staging_target.exists():
staging_target.mkdir()
console.debug("Copying build zip to {}".format(staging_target))
staging_zip = staging_target / (domain + '.zip')
try:
zipname.rename(staging_zip)
except OSError:
# rename only works within the same filesystem
# Using the shell version, as using copy + chmod doesn't always work depending on the filesystem
utils.run('mv {} {}'.format(zipname, staging_zip), shell=True)
# Provide the AWS Region and Credentials (for S3 upload) via environmental variables
env_extras = { 'AWS_REGION': bosslet_config.REGION,
'AWS_DEFAULT_REGION': bosslet_config.REGION }
if container_executable is None:
BUILD_ARGS['PREFIX'] = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files')
CMD = BUILD_CMD.format(**BUILD_ARGS)
if bosslet_config.PROFILE is not None:
env_extras['AWS_PROFILE'] = bosslet_config.PROFILE
console.info("calling build lambda on localhost")
else:
BUILD_ARGS['PREFIX'] = '/var/task'
CMD = BUILD_CMD.format(**BUILD_ARGS)
CMD = CONTAINER_CMD.format(EXECUTABLE = container_executable,
HOST_DIR = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files'),
RUNTIME = lambda_config['runtime'],
CMD = CMD)
if bosslet_config.PROFILE is not None:
# Cannot set the profile as the container will not have the credentials file
# So extract the underlying keys and provide those instead
creds = bosslet_config.session.get_credentials()
env_extras['AWS_ACCESS_KEY_ID'] = creds.access_key
env_extras['AWS_SECRET_ACCESS_KEY'] = creds.secret_key
console.info("calling build lambda in {}".format(container_executable))
try:
utils.run(CMD, env_extras=env_extras)
except Exception as ex:
raise BossManageError("Problem building {} lambda code zip: {}".format(lambda_dir, ex))
finally:
os.remove(staging_zip)
else:
BUILD_ARGS['PREFIX'] = '~'
CMD = BUILD_CMD.format(**BUILD_ARGS)
lambda_build_server_key = bosslet_config.LAMBDA_SERVER_KEY
lambda_build_server_key = utils.keypair_to_file(lambda_build_server_key)
ssh_target = SSHTarget(lambda_build_server_key, lambda_build_server, 22, 'ec2-user')
bastions = [bosslet_config.outbound_bastion] if bosslet_config.outbound_bastion else []
ssh = SSHConnection(ssh_target, bastions)
console.debug("Copying build zip to lambda-build-server")
target_file = '~/staging/{}.zip'.format(domain)
ret = ssh.scp(zipname, target_file, upload=True)
console.debug("scp return code: " + str(ret))
os.remove(zipname)
console.info("calling build lambda on lambda-build-server")
ret = ssh.cmd(CMD)
if ret != 0:
raise BossManageError("Problem building {} lambda code zip: Return code: {}".format(lambda_dir, ret))
"""
import boto3
import configparser
import io
from lib.cloudformation import CloudFormationConfiguration, Arg, Arn, Ref
from lib import aws
from lib import constants as const
from lib import lambdas
import os
import json
import shutil
import subprocess
# Location of repo with the lambda autoscaler.
LAMBDA_ROOT_FOLDER = const.repo_path('cloud_formation', 'lambda',
'dynamodb-lambda-autoscale')
CONFIG_OUTPUT_PATH = os.path.join(LAMBDA_ROOT_FOLDER, 'config.env.production')
CONFIG_INPUT_DIR = const.repo_path('cloud_formation', 'dynamodb-autoscale')
CONFIG_OUTPUT_DIR = os.path.join(LAMBDA_ROOT_FOLDER, 'src', 'configuration')
PROVISIONER_FILENAME = 'BossProvisioners.json'
DYNAMO_LAMBDA_KEY = 'DynamoLambda'
TRIGGER_KEY = 'TriggerDynamoAutoscale'
def create_config(bosslet_config):
session = bosslet_config.session
names = bosslet_config.names
config = CloudFormationConfiguration("dynamolambda", bosslet_config)
def export_path(bosslet_config):
return const.repo_path('vault', 'private', bosslet_config.names.vault.dns,
'export.json')
import alter_path
from lib.constants import repo_path
from lib import configuration
from lib import utils
CONFIGS = [
"activities",
"auth",
"backup",
"cachemanager",
"endpoint",
"vault",
]
os.environ["PATH"] += ":" + repo_path("bin") # allow executing Packer from the bin/ directory
def lambda_ami():
# Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type
# Should match runtime used by AWS Lambda
# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
# NOTE: The boss-manage code assumes that this will be an Amazon AMI
# that uses the 'ec2-user' user account
return "ami-0080e4c5bc078760e"
def get_commit():
"""Figure out the commit hash of the current git revision.
Note: Only works if the CWD is a git repository
Returns:
(string) : The git commit hash
"""
