Example #1
0
    def create(self):
        if super().create() is False:
            return False

        thresholds = self.get_thresholds()
        if thresholds is None:
            return False

        currency = self.bosslet_config.BILLING_CURRENCY
        alarm_parms = {
            'AlarmName': None,
            'AlarmDescription': None,
            'ActionsEnabled': True,
            'OKActions': [],
            'AlarmActions': [self.arn],
            'InsufficientDataActions': [],
            'MetricName': 'EstimatedCharges',
            'Namespace': 'AWS/Billing',
            'Statistic': 'Maximum',
            'Dimensions': [{'Name': 'Currency', 'Value': currency}],
            'Period': 32400,  # This should be at least 21600 (6 hrs) or all alarms will reset and fire every 6 hrs, I read 9 hours 32400 is a good number allow for billing metric data is ever slightly delayed.
            'EvaluationPeriods': 1,
            'Threshold': None,
            'ComparisonOperator': 'GreaterThanOrEqualToThreshold'
        }

        for threshold in thresholds:
            console.debug("\tAlert level: {:,}".format(threshold))
            alarm_parms['AlarmName'] = "Billing_{}_Acc#_{}".format(str(threshold), str(self.bosslet_config.ACCOUNT_ID))
            alarm_parms['AlarmDescription'] = "Alarm when spending reaches {:,}".format(threshold)
            alarm_parms['Threshold'] = float(threshold)
            response = self.client_cw.put_metric_alarm(**alarm_parms)
Example #2
0
    def save_tags(self):
        if self.min == 0 and \
           self.max == 0 and \
           self.desired == 0:
            console.debug("{} already turned off")
            return

        values = {'min': self.min, 'max': self.max, 'desired': self.desired}
        self.write_tags(values)
Example #3
0
 def call(*args, **kwargs):
     """Standin for the requested function that will print the function and arguments"""
     args_kwargs = ", ".join([
         *[pformat_truncate(arg) for arg in args], *[
             "{} = {}".format(k, pformat_truncate(v))
             for k, v in kwargs.items()
         ]
     ])
     console.debug("{}{}({})".format(self.prefix, function,
                                     args_kwargs))
def build_dependency_graph(action, bosslet_config, modules):
    """
    Given the list of bossDB modules (CloudFormation Stacks) to be installed and the action (Ex: create, update, delete)
    figure, create a dependency graph and then order the modules in the correct order (It reverses the order for deletes)
    Args:
        action(str):
        bosslet_config(BossConfiguration): bosslet_config based on a configuration file like hiderrt.boss or bossdb.boss
        modules(list[(str,module)]: List of tuples (cloudformation stack, module)

    Returns:

    """
    class Node(object):
        """Directed Dependency Graph Node"""
        def __init__(self, name):
            self.name = name
            self.edges = []

        def __repr__(self):
            return "<Node: {}>".format(self.name)

        def depends_on(self, node):
            self.edges.append(node)

    def resolve(node, resolved, seen=None):
        """From a root node, add all sub elements and then the root"""
        if seen is None:
            seen = []
        seen.append(node)
        for edge in node.edges:
            if edge not in resolved:
                if edge in seen:
                    raise exceptions.CircularDependencyError(node.name, edge.name)
                resolve(edge, resolved, seen)
        resolved.append(node)

    nums = {} # Mapping of config to index in modules list
    nodes = {} # Mapping of config to Node
    no_deps = [] # List of configs that are not the target of a dependency
                 # meaning that they are the root of a dependency tree
    # Populate variables
    for i in range(len(modules)):
        config = modules[i][0]
        nums[config] = i
        nodes[config] = Node(config)
        no_deps.append(config)

    # lookup the existing stacks to so we can verify that all dependencies will
    # be satisfied (by either existing or being launched)
    existing = { k : v['StackStatus']
                 for k, v in aws.get_existing_stacks(bosslet_config).items() }

    stop = False
    for name, status in existing.items():
        if status.endswith('_IN_PROGRESS'):
            console.warning("Config '{}' is in progress".format(name))
            stop = True
        elif status.endswith('_FAILED'):
            if name not in nodes:
                console.fail("Config '{}' is failed and should be acted upon".format(name))
                stop = True
            else:
                if action == 'delete':
                    console.info("Config '{}' is failed, deleting".format(name))
                elif status == 'UPDATE_ROLLBACK_FAILED':
                    console.fail("Config '{}' needs to be manually resolved in the AWS console".format(name))
                    stop = True
                else: # CREATE, DELETE, or ROLLBACK FAILED
                    console.fail("Config '{}' is failed, needs to be deleted".format(name))
                    stop = True
    if stop:
        raise exceptions.BossManageError('Problems with existing stacks')

    # Create dependency graph and locate root nodes
    for config, module in modules:
        deps = module.__dict__.get('DEPENDENCIES')
        if deps is None:
            continue
        if type(deps) == str:
            deps = [deps]

        for dep in deps:
            if dep not in nodes:
                # dependency not part of configs to launch
                if dep not in existing and action == "create":
                    raise exceptions.MissingDependencyError(config, dep)
            else:
                # If action is update, post-init, pre-init, verify that
                # the config is already existing
                if action not in ('create', 'delete') and dep not in existing:
                    raise exceptions.MissingDependencyError(config, dep)

                nodes[config].depends_on(nodes[dep])

                try:
                    no_deps.remove(dep)
                except:
                    pass # Doesn't exist in no_deps list

    # Resolve dependency graph
    resolved = []
    for no_dep in no_deps: # Don't have any dependencies
        resolve(nodes[no_dep], resolved)

    # Reorder input
    reordered = [ modules[nums[node.name]] for node in resolved ]

    # Extra check
    if len(reordered) != len(modules):
        raise exceptions.CircularDependencyError()

    # Removed configs that don't need to be created
    if action == "create":
        for config, module in reordered[:]:
            if config in existing:
                # If the config already exists, don't try to create it again
                console.debug('Not executing create on {} as it already exists'.format(config))
                reordered.remove((config, module))

    # Remove configs that don't need to be deleted / updated / etc
    else:
        for config, module in reordered[:]:
            if config not in existing:
                # If the config doesn't exist, don't try to delete it again
                # If the config doesn't exist, don't try to update it
                console.debug("Not executing {} on {} as it doesn't exist".format(action, config))
                reordered.remove((config, module))

        if action == "delete": # Delete in reverse order
            reordered.reverse()

    # Make sure that the target configs are not currently being processed
    for config, module in reordered:
        if config in existing and existing[config].endswith("_IN_PROGRESS"):
            raise exceptions.DependencyInProgressError(config)

    return reordered
Example #5
0
def load_lambdas_on_s3(bosslet_config, lambda_name = None, lambda_dir = None):
    """Package up the lambda files and send them through the lambda build process
    where the lambda code zip is produced and uploaded to S3

    NOTE: This function is also used to build lambda layer code zips, the only requirement
          for a layer is that the files in the resulting zip should be in the correct
          subdirectory (`python/` for Python libraries) so that when a lambda uses the
          layer the libraries included in the layer can be correctly loaded

    NOTE: If lambda_name and lambda_dir are both None then lambda_dir is set to
          'multi_lambda' for backwards compatibility

    Args:
        bosslet_config (BossConfiguration): Configuration object of the stack the
                                            lambda will be deployed into
        lambda_name (str): Name of the lambda, which will be mapped to the name of the
                           lambda directory that contains the lambda's code
        lambda_dir (str): Name of the directory in `cloud_formation/lambda/` that
                          contains the `lambda.yml` configuration file for the lambda

    Raises:
        BossManageError: If there was a problem with building the lambda code zip or
                         uploading it to the given S3 bucket
    """
    # For backwards compatibility build the multi_lambda code zip
    if lambda_name is None and lambda_dir is None:
        lambda_dir = 'multi_lambda'

    # Map from lambda_name to lambda_dir if needed
    if lambda_dir is None:
        try:
            lambda_dir = lambda_dirs(bosslet_config)[lambda_name]
        except KeyError:
            console.error("Cannot build a lambda that doesn't use a code zip file")
            return None

    # To prevent rubuilding a lambda code zip multiple times during an individual execution memorize what has been built
    if lambda_dir in BUILT_ZIPS:
        console.debug('Lambda code {} has already be build recently, skipping...'.format(lambda_dir))
        return
    BUILT_ZIPS.append(lambda_dir)

    lambda_dir = pathlib.Path(const.repo_path('cloud_formation', 'lambda', lambda_dir))
    lambda_config = lambda_dir / 'lambda.yml'
    with lambda_config.open() as fh:
        lambda_config = yaml.full_load(fh.read())

    if lambda_config.get('layers'):
        for layer in lambda_config['layers']:
            # Layer names should end with `layer`
            if not layer.endswith('layer'):
                console.warning("Layer '{}' doesn't conform to naming conventions".format(layer))

            load_lambdas_on_s3(bosslet_config, lambda_dir=layer)

    console.debug("Building {} lambda code zip".format(lambda_dir))

    domain = bosslet_config.INTERNAL_DOMAIN
    tempname = tempfile.NamedTemporaryFile(delete=True)
    zipname = pathlib.Path(tempname.name + '.zip')
    tempname.close()
    console.debug('Using temp zip file: {}'.format(zipname))

    cwd = os.getcwd()

    # Copy the lambda files into the zip
    for filename in lambda_dir.glob('*'):
        zip.write_to_zip(str(filename), zipname, arcname=filename.name)

    # Copy the other files that should be included
    if lambda_config.get('include'):
        for src in lambda_config['include']:
            dst = lambda_config['include'][src]
            src_path, src_file = src.rsplit('/', 1)

            os.chdir(const.repo_path(src_path))

            # Generate dynamic configuration files, as needed
            if src_file == 'ndingest.git':
                with open(NDINGEST_SETTINGS_TEMPLATE, 'r') as tmpl:
                    # Generate settings.ini file for ndingest.
                    create_ndingest_settings(bosslet_config, tmpl)

            zip.write_to_zip(src_file, zipname, arcname=dst)
            os.chdir(cwd)

    # Currently any Docker CLI compatible container setup can be used (like podman)
    CONTAINER_CMD = '{EXECUTABLE} run --rm -it --env AWS_* --volume {HOST_DIR}:/var/task/ lambci/lambda:build-{RUNTIME} {CMD}'

    BUILD_CMD = 'python3 {PREFIX}/build_lambda.py {DOMAIN} {BUCKET}'
    BUILD_ARGS = {
        'DOMAIN': domain,
        'BUCKET': bosslet_config.LAMBDA_BUCKET,
    }

    # DP NOTE: not sure if this should be in the bosslet_config, as it is more about the local dev
    #          environment instead of the stack's environment. Different maintainer may have different
    #          container commands installed.
    container_executable = os.environ.get('LAMBDA_BUILD_CONTAINER')
    lambda_build_server = bosslet_config.LAMBDA_SERVER
    if lambda_build_server is None:
        staging_target = pathlib.Path(const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files', 'staging'))
        if not staging_target.exists():
            staging_target.mkdir()

        console.debug("Copying build zip to {}".format(staging_target))
        staging_zip = staging_target / (domain + '.zip')
        try:
            zipname.rename(staging_zip)
        except OSError:
            # rename only works within the same filesystem
            # Using the shell version, as using copy +  chmod doesn't always work depending on the filesystem
            utils.run('mv {} {}'.format(zipname, staging_zip), shell=True)

        # Provide the AWS Region and Credentials (for S3 upload) via environmental variables
        env_extras = { 'AWS_REGION': bosslet_config.REGION,
                       'AWS_DEFAULT_REGION': bosslet_config.REGION }

        if container_executable is None:
            BUILD_ARGS['PREFIX'] = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files')
            CMD = BUILD_CMD.format(**BUILD_ARGS)

            if bosslet_config.PROFILE is not None:
                env_extras['AWS_PROFILE'] = bosslet_config.PROFILE

            console.info("calling build lambda on localhost")
        else:
            BUILD_ARGS['PREFIX'] = '/var/task'
            CMD = BUILD_CMD.format(**BUILD_ARGS)
            CMD = CONTAINER_CMD.format(EXECUTABLE = container_executable,
                                       HOST_DIR = const.repo_path('salt_stack', 'salt', 'lambda-dev', 'files'),
                                       RUNTIME = lambda_config['runtime'],
                                       CMD = CMD)

            if bosslet_config.PROFILE is not None:
                # Cannot set the profile as the container will not have the credentials file
                # So extract the underlying keys and provide those instead
                creds = bosslet_config.session.get_credentials()
                env_extras['AWS_ACCESS_KEY_ID'] = creds.access_key
                env_extras['AWS_SECRET_ACCESS_KEY'] = creds.secret_key

            console.info("calling build lambda in {}".format(container_executable))

        try:
            utils.run(CMD, env_extras=env_extras)
        except Exception as ex:
            raise BossManageError("Problem building {} lambda code zip: {}".format(lambda_dir, ex))
        finally:
            os.remove(staging_zip)

    else:
        BUILD_ARGS['PREFIX'] = '~'
        CMD = BUILD_CMD.format(**BUILD_ARGS)

        lambda_build_server_key = bosslet_config.LAMBDA_SERVER_KEY
        lambda_build_server_key = utils.keypair_to_file(lambda_build_server_key)
        ssh_target = SSHTarget(lambda_build_server_key, lambda_build_server, 22, 'ec2-user')
        bastions = [bosslet_config.outbound_bastion] if bosslet_config.outbound_bastion else []
        ssh = SSHConnection(ssh_target, bastions)

        console.debug("Copying build zip to lambda-build-server")
        target_file = '~/staging/{}.zip'.format(domain)
        ret = ssh.scp(zipname, target_file, upload=True)
        console.debug("scp return code: " + str(ret))

        os.remove(zipname)

        console.info("calling build lambda on lambda-build-server")
        ret = ssh.cmd(CMD)
        if ret != 0:
            raise BossManageError("Problem building {} lambda code zip: Return code: {}".format(lambda_dir, ret))
Example #6
0
                        running[name] = inst
            kwargs['NextToken'] = resp.get('NextToken')

        if len(running) == 0:
            console.info("No unexpected running EC2 instances exist")

        if not args.quiet:
            for inst in unlabled:
                console.warning('Instance {} is not labeled'.format(
                    inst['InstanceId']))
            for name, inst in items(stopped):
                console.warning('Instance {} ({}) is stopped'.format(
                    inst['InstanceId'], name))
        if args.delete:
            for name, inst in items(running):
                console.debug('Deleting EC2 instance {} ({})'.format(
                    inst['InstanceId'], name))
            if console.confirm('Are you sure', default=False):
                for inst in running.values():
                    client.terminate_instances(
                        InstanceIds=[inst['InstanceId']])
        else:
            for name, inst in items(running):
                console.debug('Instance {} ({})'.format(
                    inst['InstanceId'], name))
    elif args.resource == 'sg':
        client = args.bosslet_config.session.client('ec2')

        packer_sgs = []
        launch_sgs = []

        kwargs = {}