def create(self):
if super().create() is False:
return False
thresholds = self.get_thresholds()
if thresholds is None:
return False
currency = self.bosslet_config.BILLING_CURRENCY
alarm_parms = {
'AlarmName': None,
'AlarmDescription': None,
'ActionsEnabled': True,
'OKActions': [],
'AlarmActions': [self.arn],
'InsufficientDataActions': [],
'MetricName': 'EstimatedCharges',
'Namespace': 'AWS/Billing',
'Statistic': 'Maximum',
'Dimensions': [{'Name': 'Currency', 'Value': currency}],
'Period': 32400, # This should be at least 21600 (6 hrs) or all alarms will reset and fire every 6 hrs, I read 9 hours 32400 is a good number allow for billing metric data is ever slightly delayed.
'EvaluationPeriods': 1,
'Threshold': None,
'ComparisonOperator': 'GreaterThanOrEqualToThreshold'
}
for threshold in thresholds:
console.debug("\tAlert level: {:,}".format(threshold))
alarm_parms['AlarmName'] = "Billing_{}_Acc#_{}".format(str(threshold), str(self.bosslet_config.ACCOUNT_ID))
alarm_parms['AlarmDescription'] = "Alarm when spending reaches {:,}".format(threshold)
alarm_parms['Threshold'] = float(threshold)
response = self.client_cw.put_metric_alarm(**alarm_parms)
def save_tags(self):
if self.min == 0 and \
self.max == 0 and \
self.desired == 0:
console.debug("{} already turned off")
return
values = {'min': self.min, 'max': self.max, 'desired': self.desired}
self.write_tags(values)
def call(*args, **kwargs):
"""Standin for the requested function that will print the function and arguments"""
args_kwargs = ", ".join([
*[pformat_truncate(arg) for arg in args], *[
"{} = {}".format(k, pformat_truncate(v))
for k, v in kwargs.items()
]
])
console.debug("{}{}({})".format(self.prefix, function,
args_kwargs))
def build_dependency_graph(action, bosslet_config, modules):
"""
Given the list of bossDB modules (CloudFormation Stacks) to be installed and the action (Ex: create, update, delete)
figure, create a dependency graph and then order the modules in the correct order (It reverses the order for deletes)
Args:
action(str):
bosslet_config(BossConfiguration): bosslet_config based on a configuration file like hiderrt.boss or bossdb.boss
modules(list[(str,module)]: List of tuples (cloudformation stack, module)
Returns:
"""
class Node(object):
"""Directed Dependency Graph Node"""
def __init__(self, name):
self.name = name
self.edges = []
def __repr__(self):
return "<Node: {}>".format(self.name)
def depends_on(self, node):
self.edges.append(node)
def resolve(node, resolved, seen=None):
"""From a root node, add all sub elements and then the root"""
if seen is None:
seen = []
seen.append(node)
for edge in node.edges:
if edge not in resolved:
if edge in seen:
raise exceptions.CircularDependencyError(node.name, edge.name)
resolve(edge, resolved, seen)
resolved.append(node)
nums = {} # Mapping of config to index in modules list
nodes = {} # Mapping of config to Node
no_deps = [] # List of configs that are not the target of a dependency
# meaning that they are the root of a dependency tree
# Populate variables
for i in range(len(modules)):
config = modules[i][0]
nums[config] = i
nodes[config] = Node(config)
no_deps.append(config)
# lookup the existing stacks to so we can verify that all dependencies will
# be satisfied (by either existing or being launched)
existing = { k : v['StackStatus']
for k, v in aws.get_existing_stacks(bosslet_config).items() }
stop = False
for name, status in existing.items():
if status.endswith('_IN_PROGRESS'):
console.warning("Config '{}' is in progress".format(name))
stop = True
elif status.endswith('_FAILED'):
if name not in nodes:
console.fail("Config '{}' is failed and should be acted upon".format(name))
stop = True
else:
if action == 'delete':
console.info("Config '{}' is failed, deleting".format(name))
elif status == 'UPDATE_ROLLBACK_FAILED':
console.fail("Config '{}' needs to be manually resolved in the AWS console".format(name))
stop = True
else: # CREATE, DELETE, or ROLLBACK FAILED
console.fail("Config '{}' is failed, needs to be deleted".format(name))
stop = True
if stop:
raise exceptions.BossManageError('Problems with existing stacks')
# Create dependency graph and locate root nodes
for config, module in modules:
deps = module.__dict__.get('DEPENDENCIES')
if deps is None:
continue
if type(deps) == str:
deps = [deps]
for dep in deps:
if dep not in nodes:
# dependency not part of configs to launch
if dep not in existing and action == "create":
raise exceptions.MissingDependencyError(config, dep)
else:
# If action is update, post-init, pre-init, verify that
# the config is already existing
if action not in ('create', 'delete') and dep not in existing:
raise exceptions.MissingDependencyError(config, dep)
nodes[config].depends_on(nodes[dep])
try:
no_deps.remove(dep)
except:
pass # Doesn't exist in no_deps list
# Resolve dependency graph
resolved = []
for no_dep in no_deps: # Don't have any dependencies
resolve(nodes[no_dep], resolved)
# Reorder input
reordered = [ modules[nums[node.name]] for node in resolved ]
# Extra check
if len(reordered) != len(modules):
raise exceptions.CircularDependencyError()
# Removed configs that don't need to be created
if action == "create":
for config, module in reordered[:]:
if config in existing:
# If the config already exists, don't try to create it again
console.debug('Not executing create on {} as it already exists'.format(config))
reordered.remove((config, module))
# Remove configs that don't need to be deleted / updated / etc
else:
for config, module in reordered[:]:
if config not in existing:
# If the config doesn't exist, don't try to delete it again
# If the config doesn't exist, don't try to update it
console.debug("Not executing {} on {} as it doesn't exist".format(action, config))
reordered.remove((config, module))
if action == "delete": # Delete in reverse order
reordered.reverse()
# Make sure that the target configs are not currently being processed
for config, module in reordered:
if config in existing and existing[config].endswith("_IN_PROGRESS"):
raise exceptions.DependencyInProgressError(config)
return reordered
def load_lambdas_on_s3(bosslet_config, lambda_name = None, lambda_dir = None):
"""Package up the lambda files and send them through the lambda build process
where the lambda code zip is produced and uploaded to S3
NOTE: This function is also used to build lambda layer code zips, the only requirement
for a layer is that the files in the resulting zip should be in the correct
subdirectory (`python/` for Python libraries) so that when a lambda uses the
layer the libraries included in the layer can be correctly loaded
NOTE: If lambda_name and lambda_dir are both None then lambda_dir is set to
'multi_lambda' for backwards compatibility
Args:
bosslet_config (BossConfiguration): Configuration object of the stack the
lambda will be deployed into
lambda_name (str): Name of the lambda, which will be mapped to the name of the
lambda directory that contains the lambda's code
lambda_dir (str): Name of the directory in `cloud_formation/lambda/` that
contains the `lambda.yml` configuration file for the lambda
Raises:
BossManageError: If there was a problem with building the lambda code zip or
uploading it to the given S3 bucket
"""
# For backwards compatibility build the multi_lambda code zip
if lambda_name is None and lambda_dir is None:
lambda_dir = 'multi_lambda'
# Map from lambda_name to lambda_dir if needed
if lambda_dir is None:
try:
lambda_dir = lambda_dirs(bosslet_config)[lambda_name]
except KeyError:
console.error("Cannot build a lambda that doesn't use a code zip file")
return None
# To prevent rubuilding a lambda code zip multiple times during an individual execution memorize what has been built
if lambda_dir in BUILT_ZIPS:
console.debug('Lambda code {} has already be build recently, skipping...'.format(lambda_dir))
return
BUILT_ZIPS.append(lambda_dir)
lambda_dir = pathlib.Path(const.repo_path('cloud_formation', 'lambda', lambda_dir))
lambda_config = lambda_dir / 'lambda.yml'
with lambda_config.open() as fh:
lambda_config = yaml.full_load(fh.read())
if lambda_config.get('layers'):
for layer in lambda_config['layers']:
# Layer names should end with `layer`
if not layer.endswith('layer'):
console.warning("Layer '{}' doesn't conform to naming conventions".format(layer))
load_lambdas_on_s3(bosslet_config, lambda_dir=layer)
console.debug("Building {} lambda code zip".format(lambda_dir))
domain = bosslet_config.INTERNAL_DOMAIN
tempname = tempfile.NamedTemporaryFile(delete=True)
zipname = pathlib.Path(tempname.name + '.zip')
tempname.close()
console.debug('Using temp zip file: {}'.format(zipname))
cwd = os.getcwd()
# Copy the lambda files into the zip
for filename in lambda_dir.glob('*'):
zip.write_to_zip(str(filename), zipname, arcname=filename.name)
# Copy the other files that should be included
if lambda_config.get('include'):
for src in lambda_config['include']:
dst = lambda_config['include'][src]
src_path, src_file = src.rsplit('/', 1)
os.chdir(const.repo_path(src_path))
# Generate dynamic configuration files, as needed
if src_file == 'ndingest.git':
with open(NDINGEST_SETTINGS_TEMPLATE, 'r') as tmpl:
# Generate settings.ini file for ndingest.
create_ndingest_settings(bosslet_config, tmpl)
zip.write_to_zip(src_file, zipname, arcname=dst)
os.chdir(cwd)
# Currently any Docker CLI compatible container setup can be used (like podman)
CONTAINER_CMD = '{EXECUTABLE} run --rm -it --env AWS_* --volume {HOST_DIR}:/var/task/ lambci/lambda:build-{RUNTIME} {CMD}'
BUILD_CMD = 'python3 {PREFIX}/build_lambda.py {DOMAIN} {BUCKET}'
BUILD_ARGS = {
'DOMAIN': domain,
'BUCKET': bosslet_config.LAMBDA_BUCKET,
}
# DP NOTE: not sure if this should be in the bosslet_config, as it is more about the local dev
# environment instead of the stack's environment. Different maintainer may have different
# container commands installed.
container_executable = os.environ.get('LAMBDA_BUILD_CONTAINER')
lambda_build_server = bosslet_config.LAMBDA_SERVER
if lambda_build_server is None:
staging_target = pathlib.Path(const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files', 'staging'))
if not staging_target.exists():
staging_target.mkdir()
console.debug("Copying build zip to {}".format(staging_target))
staging_zip = staging_target / (domain + '.zip')
try:
zipname.rename(staging_zip)
except OSError:
# rename only works within the same filesystem
# Using the shell version, as using copy + chmod doesn't always work depending on the filesystem
utils.run('mv {} {}'.format(zipname, staging_zip), shell=True)
# Provide the AWS Region and Credentials (for S3 upload) via environmental variables
env_extras = { 'AWS_REGION': bosslet_config.REGION,
'AWS_DEFAULT_REGION': bosslet_config.REGION }
if container_executable is None:
BUILD_ARGS['PREFIX'] = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files')
CMD = BUILD_CMD.format(**BUILD_ARGS)
if bosslet_config.PROFILE is not None:
env_extras['AWS_PROFILE'] = bosslet_config.PROFILE
console.info("calling build lambda on localhost")
else:
BUILD_ARGS['PREFIX'] = '/var/task'
CMD = BUILD_CMD.format(**BUILD_ARGS)
CMD = CONTAINER_CMD.format(EXECUTABLE = container_executable,
HOST_DIR = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files'),
RUNTIME = lambda_config['runtime'],
CMD = CMD)
if bosslet_config.PROFILE is not None:
# Cannot set the profile as the container will not have the credentials file
# So extract the underlying keys and provide those instead
creds = bosslet_config.session.get_credentials()
env_extras['AWS_ACCESS_KEY_ID'] = creds.access_key
env_extras['AWS_SECRET_ACCESS_KEY'] = creds.secret_key
console.info("calling build lambda in {}".format(container_executable))
try:
utils.run(CMD, env_extras=env_extras)
except Exception as ex:
raise BossManageError("Problem building {} lambda code zip: {}".format(lambda_dir, ex))
finally:
os.remove(staging_zip)
else:
BUILD_ARGS['PREFIX'] = '~'
CMD = BUILD_CMD.format(**BUILD_ARGS)
lambda_build_server_key = bosslet_config.LAMBDA_SERVER_KEY
lambda_build_server_key = utils.keypair_to_file(lambda_build_server_key)
ssh_target = SSHTarget(lambda_build_server_key, lambda_build_server, 22, 'ec2-user')
bastions = [bosslet_config.outbound_bastion] if bosslet_config.outbound_bastion else []
ssh = SSHConnection(ssh_target, bastions)
console.debug("Copying build zip to lambda-build-server")
target_file = '~/staging/{}.zip'.format(domain)
ret = ssh.scp(zipname, target_file, upload=True)
console.debug("scp return code: " + str(ret))
os.remove(zipname)
console.info("calling build lambda on lambda-build-server")
ret = ssh.cmd(CMD)
if ret != 0:
raise BossManageError("Problem building {} lambda code zip: Return code: {}".format(lambda_dir, ret))
running[name] = inst
kwargs['NextToken'] = resp.get('NextToken')
if len(running) == 0:
console.info("No unexpected running EC2 instances exist")
if not args.quiet:
for inst in unlabled:
console.warning('Instance {} is not labeled'.format(
inst['InstanceId']))
for name, inst in items(stopped):
console.warning('Instance {} ({}) is stopped'.format(
inst['InstanceId'], name))
if args.delete:
for name, inst in items(running):
console.debug('Deleting EC2 instance {} ({})'.format(
inst['InstanceId'], name))
if console.confirm('Are you sure', default=False):
for inst in running.values():
client.terminate_instances(
InstanceIds=[inst['InstanceId']])
else:
for name, inst in items(running):
console.debug('Instance {} ({})'.format(
inst['InstanceId'], name))
elif args.resource == 'sg':
client = args.bosslet_config.session.client('ec2')
packer_sgs = []
launch_sgs = []
kwargs = {}