示例#1
0
    def test_policy_region_expand_global(self):
        factory = self.replay_flight_data('test_aws_policy_global_expand')
        self.patch(aws, '_profile_session', factory())
        original = self.policy_loader.load_data(
            {"policies": [
                {"name": "foo", "resource": "s3"},
                {"name": "iam", "resource": "iam-user"}]},
            'memory://',
            config=Config.empty(regions=["us-east-1", "us-west-2"]),
        )

        collection = AWS().initialize_policies(original, Config.empty(regions=["all"]))
        self.assertEqual(len(collection.resource_types), 2)
        s3_regions = [p.options.region for p in collection if p.resource_type == "s3"]
        self.assertTrue("us-east-1" in s3_regions)
        self.assertTrue("us-east-2" in s3_regions)
        iam = [p for p in collection if p.resource_type == "iam-user"]
        self.assertEqual(len(iam), 1)
        self.assertEqual(iam[0].options.region, "us-east-1")

        collection = AWS().initialize_policies(
            original, Config.empty(regions=["eu-west-1", "eu-west-2"])
        )
        iam = [p for p in collection if p.resource_type == "iam-user"]
        self.assertEqual(len(iam), 1)
        self.assertEqual(iam[0].options.region, "eu-west-1")
        self.assertEqual(len(collection), 3)
示例#2
0
 def load_policy_set(self, data, config=None):
     filename = self.write_policy_file(data, format="json")
     if config:
         e = Config.empty(**config)
     else:
         e = Config.empty()
     return policy.load(e, filename)
示例#3
0
 def load_policy_set(self, data, config=None):
     filename = self.write_policy_file(data)
     if config:
         e = Config.empty(**config)
     else:
         e = Config.empty()
     return policy.load(e, filename)
示例#4
0
 def load_policy_set(self, data, config=None):
     filename = self.write_policy_file(data)
     if config:
         config['account_id'] = ACCOUNT_ID
         e = Config.empty(**config)
     else:
         e = Config.empty(account_id=ACCOUNT_ID)
     return policy.load(e, filename)
示例#5
0
def main():
    parser = setup_parser()
    argcomplete.autocomplete(parser)
    options = parser.parse_args()
    if options.subparser is None:
        parser.print_help(file=sys.stderr)
        return sys.exit(2)

    _setup_logger(options)

    # Support the deprecated -c option
    if getattr(options, 'config', None) is not None:
        options.configs.append(options.config)

    config = Config.empty(**vars(options))

    try:
        command = options.command
        if not callable(command):
            command = getattr(
                importlib.import_module(command.rsplit('.', 1)[0]),
                command.rsplit('.', 1)[-1])

        # Set the process name to something cleaner
        process_name = [os.path.basename(sys.argv[0])]
        process_name.extend(sys.argv[1:])
        setproctitle(' '.join(process_name))
        command(config)
    except Exception:
        if not options.debug:
            raise
        traceback.print_exc()
        pdb.post_mortem(sys.exc_info()[-1])
示例#6
0
def tag(assume, region, db, creator_tag, user_suffix, dryrun,
        summary=True, profile=None, type=()):
    """Tag resources with their creator.
    """
    trail_db = TrailDB(db)
    load_resources()

    with temp_dir() as output_dir:
        config = ExecConfig.empty(
            output_dir=output_dir, assume=assume,
            region=region, profile=profile)
        factory = aws.AWS().get_session_factory(config)
        account_id = local_session(factory).client('sts').get_caller_identity().get('Account')
        config['account_id'] = account_id
        tagger = ResourceTagger(trail_db, config, creator_tag, user_suffix, dryrun, type)

        try:
            stats = tagger.process()
        except Exception:
            log.exception(
                "error processing account:%s region:%s config:%s env:%s",
                account_id, region, config, dict(os.environ))
            raise

    if not summary:
        return stats

    log.info(
        "auto tag summary account:%s region:%s \n%s",
        config['account_id'],
        config['region'],
        "\n".join([" {}: {}".format(k, v) for k, v in stats.items() if v]))
    total = sum([v for k, v in stats.items() if not k.endswith('not-found')])
    log.info("Total resources tagged: %d" % total)
示例#7
0
def test_cloudtrail_policy():
    collection = PolicyLoader(Config.empty()).load_data(
        {
            'policies': [{
                'name': 'check-ec2',
                'resource': 'ec2',
                'mode': {
                    'type': 'cloudtrail',
                    'events': ['RunInstances']
                }
            }]
        },
        file_uri=":mem:")
    sam = {'Resources': {}}
    p = list(collection).pop()
    dispatch_render(p, sam)
    assert sam['Resources']['CheckEc2']['Properties']['Events'] == {
        'PolicyTriggerA': {
            'Properties': {
                'Pattern': {
                    'detail': {
                        'eventName': ['RunInstances'],
                        'eventSource': ['ec2.amazonaws.com']
                    },
                    'detail-type': ['AWS API Call via CloudTrail']
                }
            },
            'Type': 'CloudWatchEvent'
        }
    }
示例#8
0
def run(event, context=None):
    # policies file should always be valid in functions so do loading naively
    with open('config.json') as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        log.error('Invalid policy config')
        return False

    options_overrides = \
        policy_config['policies'][0].get('mode', {}).get('execution-options', {})

    # if output_dir specified use that, otherwise make a temp directory
    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = get_tmp_output_dir()

    # merge all our options in
    options = Config.empty(**options_overrides)

    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            log.info("running policy %s", p.name)
            p.push(event, context)
    return True
示例#9
0
 def get_dir_output(self, location):
     work_dir = self.change_cwd()
     return work_dir, DirectoryOutput(
         ExecutionContext(None, Bag(name="xyz", provider_name="ostack"),
                          Config.empty(output_dir=location)),
         {'url': location},
     )
示例#10
0
 def get_context(self, config=None, policy=None):
     if config is None:
         self.context_output_dir = self.get_temp_dir()
         config = Config.empty(output_dir=self.context_output_dir)
     ctx = ExecutionContext(Session, policy or Bag({'name': 'test-policy'}),
                            config)
     return ctx
示例#11
0
def dispatch_event(event, context):

    global account_id

    error = event.get('detail', {}).get('errorCode')
    if error:
        log.debug("Skipping failed operation: %s" % error)
        return

    event['debug'] = True
    if event['debug']:
        log.info("Processing event\n %s", format_event(event))

    # Policies file should always be valid in lambda so do loading naively
    with open('config.json') as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        return False

    # Initialize output directory, we've seen occassional perm issues with
    # lambda on temp directory and changing unix execution users, so
    # use a per execution temp space.
    output_dir = os.environ.get('C7N_OUTPUT_DIR', '/tmp/' + str(uuid.uuid4()))
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except OSError as error:
            log.warning("Unable to make output directory: {}".format(error))

    # TODO. This enshrines an assumption of a single policy per lambda.
    options_overrides = policy_config['policies'][0].get('mode', {}).get(
        'execution-options', {})

    # if using assume role in lambda ensure that the correct
    # execution account is captured in options.
    if 'assume_role' in options_overrides:
        account_id = options_overrides['assume_role'].split(':')[4]
    elif account_id is None:
        session = boto3.Session()
        account_id = get_account_id_from_sts(session)

    # Historical compatibility with manually set execution options
    # previously this was a boolean, its now a string value with the
    # boolean flag triggering a string value of 'aws'
    if 'metrics_enabled' in options_overrides and isinstance(
            options_overrides['metrics_enabled'], bool):
        options_overrides['metrics_enabled'] = 'aws'

    options_overrides['account_id'] = account_id

    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = output_dir
    options = Config.empty(**options_overrides)

    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            p.push(event, context)
    return True
示例#12
0
def report_account(account, region, policies_config, output_path, debug):
    cache_path = os.path.join(output_path, "c7n.cache")
    output_path = os.path.join(output_path, account['name'], region)
    config = Config.empty(
        region=region,
        output_dir=output_path,
        account_id=account['account_id'], metrics_enabled=False,
        cache=cache_path, log_group=None, profile=None, external_id=None)

    if account.get('role'):
        config['assume_role'] = account['role']
        config['external_id'] = account.get('external_id')
    elif account.get('profile'):
        config['profile'] = account['profile']

    policies = PolicyCollection.from_data(policies_config, config)
    records = []
    for p in policies:
        log.debug(
            "Report policy:%s account:%s region:%s path:%s",
            p.name, account['name'], region, output_path)
        policy_records = fs_record_set(p.ctx.output_path, p.name)
        for r in policy_records:
            r['policy'] = p.name
            r['region'] = p.options.region
            r['account'] = account['name']
            for t in account.get('tags', ()):
                if ':' in t:
                    k, v = t.split(':', 1)
                    r[k] = v
        records.extend(policy_records)
    return records
示例#13
0
def main():
    parser = setup_parser()
    options = parser.parse_args()
    config = Config.empty()
    resources.load_resources()

    collection = policy_load(
        config, options.config_file).filter(options.policy_filter)

    sam = {
        'AWSTemplateFormatVersion': '2010-09-09',
        'Transform': 'AWS::Serverless-2016-10-31',
        'Resources': {}}

    for p in collection:
        if p.provider_name != 'aws':
            continue
        exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
        if exec_mode_type == 'pull':
            continue

        sam_func = render(p)
        if sam_func:
            sam['Resources'][resource_name(p.name)] = sam_func
            sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
        else:
            print("unable to render sam for policy:%s" % p.name)
            continue

        archive = mu.PolicyLambda(p).get_archive()
        with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
            fh.write(archive.get_bytes())

    with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
        fh.write(yaml.safe_dump(sam, default_flow_style=False))
示例#14
0
 def dryrun(self):
     # This is largely based off of mugc.main()
     logging.getLogger('botocore').setLevel(logging.ERROR)
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     logging.getLogger('c7n.cache').setLevel(logging.WARNING)
     conf = Config.empty(
         config_files=['custodian_%s.yml' % self.region_name],
         regions=[self.region_name],
         prefix=self.config.function_prefix,
         policy_regex='^' + re.escape(self.config.function_prefix) + '.*',
         assume=None,
         policy_filter=None,
         log_group=None,
         external_id=None,
         cache_period=0,
         cache=None,
         present=False,
         dryrun=True
     )
     # use cloud provider to initialize policies to get region expansion
     policies = AWS().initialize_policies(
         PolicyCollection(
             [
                 p for p in load_policies(conf, conf)
                 if p.provider_name == 'aws'
             ],
             conf
         ),
         conf
     )
     resources_gc_prefix(conf, conf, policies)
示例#15
0
def main():
    parser = setup_parser()
    options = parser.parse_args()
    config = Config.empty()
    resources.load_resources()

    collection = policy_load(
        config, options.config_file).filter(options.policy_filter)

    sam = {
        'AWSTemplateFormatVersion': '2010-09-09',
        'Transform': 'AWS::Serverless-2016-10-31',
        'Resources': {}}

    for p in collection:
        if p.provider_name != 'aws':
            continue
        exec_mode_type = p.data.get('mode', {'type': 'pull'}).get('type')
        if exec_mode_type == 'pull':
            continue

        sam_func = render(p)
        if sam_func:
            sam['Resources'][resource_name(p.name)] = sam_func
            sam_func['Properties']['CodeUri'] = './%s.zip' % p.name
        else:
            print("unable to render sam for policy:%s" % p.name)
            continue

        archive = mu.PolicyLambda(p).get_archive()
        with open(os.path.join(options.output_dir, "%s.zip" % p.name), 'wb') as fh:
            fh.write(archive.get_bytes())

    with open(os.path.join(options.output_dir, 'deploy.yml'), 'w') as fh:
        fh.write(yaml.safe_dump(sam, default_flow_style=False))
示例#16
0
    def load_policy(
        self,
        data,
        config=None,
        session_factory=None,
        validate=C7N_VALIDATE,
        output_dir=None,
        cache=False,
    ):
        if validate:
            if not self.custodian_schema:
                self.custodian_schema = generate()
            errors = schema_validate({"policies": [data]}, self.custodian_schema)
            if errors:
                raise errors[0]

        config = config or {}
        if not output_dir:
            temp_dir = self.get_temp_dir()
            config["output_dir"] = temp_dir
        if cache:
            config["cache"] = os.path.join(temp_dir, "c7n.cache")
            config["cache_period"] = 300
        conf = Config.empty(**config)
        p = policy.Policy(data, conf, session_factory)
        p.validate()
        return p
示例#17
0
def report_account(account, region, policies_config, output_path, debug):
    cache_path = os.path.join(output_path, "c7n.cache")
    output_path = os.path.join(output_path, account['name'], region)
    config = Config.empty(
        region=region,
        output_dir=output_path,
        account_id=account['account_id'], metrics_enabled=False,
        cache=cache_path, log_group=None, profile=None, external_id=None)

    if account.get('role'):
        config['assume_role'] = account['role']
        config['external_id'] = account.get('external_id')
    elif account.get('profile'):
        config['profile'] = account['profile']

    policies = PolicyCollection.from_data(policies_config, config)
    records = []
    for p in policies:
        log.debug(
            "Report policy:%s account:%s region:%s path:%s",
            p.name, account['name'], region, output_path)
        policy_records = fs_record_set(p.ctx.output_path, p.name)
        for r in policy_records:
            r['policy'] = p.name
            r['region'] = p.options.region
            r['account'] = account['name']
            for t in account.get('tags', ()):
                if ':' in t:
                    k, v = t.split(':', 1)
                    r[k] = v
        records.extend(policy_records)
    return records
示例#18
0
def run(event, context):

    # policies file should always be valid in functions so do loading naively
    with open(context['config_file']) as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        return False

    # Initialize output directory
    output_dir = os.environ.get(
        'C7N_OUTPUT_DIR',
        '/tmp/' + str(uuid.uuid4()))
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except OSError as error:
            log.warning("Unable to make output directory: {}".format(error))

    options_overrides = \
        policy_config['policies'][0].get('mode', {}).get('execution-options', {})

    options_overrides['authorization_file'] = context['auth_file']

    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = output_dir
    options = Config.empty(**options_overrides)

    load_resources()
    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            p.push(event, context)
    return True
示例#19
0
def run(event, context):
    # policies file should always be valid in functions so do loading naively
    with open(context['config_file']) as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        log.error('Invalid policy config')
        return False

    options_overrides = \
        policy_config['policies'][0].get('mode', {}).get('execution-options', {})

    # setup our auth file location on disk
    options_overrides['authorization_file'] = context['auth_file']

    # if output_dir specified use that, otherwise make a temp directory
    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = get_tmp_output_dir()

    # merge all our options in
    options = Config.empty(**options_overrides)

    load_resources()

    options = Azure().initialize(options)

    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            try:
                p.push(event, context)
            except (CloudError, AzureHttpError) as error:
                log.error("Unable to process policy: %s :: %s" % (p.name, error))
    return True
示例#20
0
def eperm(provider, el, r=None):
    if el.permissions:
        return el.permissions
    element_type = get_element_type(el)
    if r is None or r.type is None:
        # dummy resource type for policy
        if provider == 'aws':
            r = Bag({'type': 'kinesis'})
        elif provider == 'gcp':
            r = Bag({'type': 'instance'})
        elif provider == 'azure':
            r = Bag({'type': 'vm'})

    # print(f'policy construction lookup {r.type}.{element_type}.{el.type}')

    loader = PolicyLoader(Config.empty())
    pdata = {
        'name': f'permissions-{r.type}',
        'resource': f'{provider}.{r.type}'
    }
    pdata[element_type] = get_element_data(element_type, el)

    try:
        pset = loader.load_data({'policies': [pdata]}, ':mem:', validate=False)
    except Exception as e:
        print(f'error loading {el} as {element_type}:{el.type} error: {e} \n {pdata}')
        return []
    el = get_policy_element(el, list(pset)[0])
    return el.get_permissions()
示例#21
0
def validate(options):
    load_resources()
    if len(options.configs) < 1:
        log.error('no config files specified')
        sys.exit(1)

    used_policy_names = set()
    schm = schema.generate()
    errors = []

    for config_file in options.configs:
        config_file = os.path.expanduser(config_file)
        if not os.path.exists(config_file):
            raise ValueError("Invalid path for config %r" % config_file)

        options.dryrun = True
        fmt = config_file.rsplit('.', 1)[-1]
        with open(config_file) as fh:
            if fmt in ('yml', 'yaml'):
                data = yaml.safe_load(fh.read())
            elif fmt in ('json', ):
                data = json.load(fh)
            else:
                log.error("The config file must end in .json, .yml or .yaml.")
                raise ValueError(
                    "The config file must end in .json, .yml or .yaml.")

        errors += schema.validate(data, schm)
        conf_policy_names = {
            p.get('name', 'unknown')
            for p in data.get('policies', ())
        }
        dupes = conf_policy_names.intersection(used_policy_names)
        if len(dupes) >= 1:
            errors.append(
                ValueError(
                    "Only one policy with a given name allowed, duplicates: %s"
                    % (", ".join(dupes))))
        used_policy_names = used_policy_names.union(conf_policy_names)
        if not errors:
            null_config = Config.empty(dryrun=True,
                                       account_id='na',
                                       region='na')
            for p in data.get('policies', ()):
                try:
                    policy = Policy(p, null_config, Bag())
                    policy.validate()
                except Exception as e:
                    msg = "Policy: %s is invalid: %s" % (p.get(
                        'name', 'unknown'), e)
                    errors.append(msg)
        if not errors:
            log.info("Configuration valid: {}".format(config_file))
            continue

        log.error("Configuration invalid: {}".format(config_file))
        for e in errors:
            log.error("%s" % e)
    if errors:
        sys.exit(1)
示例#22
0
 def test_policy_resource_limits_count(self):
     session_factory = self.replay_flight_data(
         "test_policy_resource_count")
     p = self.load_policy(
         {
             "name": "ecs-cluster-resource-count",
             "resource": "ecs",
             "max-resources": 1
         },
         session_factory=session_factory)
     self.assertRaises(ResourceLimitExceeded, p.run)
     policy = {
         "name": "ecs-cluster-resource-count",
         "resource": "ecs",
         "max-resources": 0
     }
     config = Config.empty(validate=True)
     self.assertRaises(
         Exception,
         self.load_policy,
         policy,
         config=config,
         validate=True,
         session_factory=session_factory
     )
示例#23
0
    def run(self):
        """
        Perform an actual run of cloud-custodian.

        This replicates the command:
        custodian run --region '${region}' --metrics -v -s \
          cloud-custodian-${account_id}-${region}/logs \
          --log-group=/cloud-custodian/${account_id}/${region} \
          -c custodian_${region}.yml \
          --cache '/tmp/.cache/cloud-custodian.cache'
        """
        conf = Config.empty(
            configs=['custodian_%s.yml' % self.region_name],
            region=self.region_name,
            regions=[self.region_name],
            log_group=self.config.custodian_log_group,
            verbose=1,
            metrics_enabled=True,
            subparser='run',
            cache='/tmp/.cache/cloud-custodian.cache',
            command='c7n.commands.run',
            output_dir='%s/logs' % self.config.output_s3_bucket_name,
            vars=None,
            dryrun=False
        )
        run(conf)
    def test_verify_parent_filter(self):
        manager = KeyVaultKeys(
            ExecutionContext(None, Bag(name="xyz", provider_name='azure'),
                             Config.empty()), {
                                 'name':
                                 'test-policy',
                                 'resource':
                                 'azure.keyvault-keys',
                                 'filters': [{
                                     'type': 'parent',
                                     'filter': {
                                         'type': 'value',
                                         'key': 'name',
                                         'op': 'glob',
                                         'value': 'cctestkv*'
                                     }
                                 }]
                             })

        self.assertEqual(len(manager.filters), 1)

        filter = manager.filters[0]
        self.assertTrue(isinstance(filter, ParentFilter))
        self.assertTrue(isinstance(filter.parent_manager, KeyVault))
        self.assertTrue(isinstance(filter.parent_filter, ValueFilter))
示例#25
0
def run(event, context=None):
    # policies file should always be valid in functions so do loading naively
    with open('config.json') as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        log.error('Invalid policy config')
        return False

    options_overrides = \
        policy_config['policies'][0].get('mode', {}).get('execution-options', {})

    # if output_dir specified use that, otherwise make a temp directory
    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = get_tmp_output_dir()

    # merge all our options in
    options = Config.empty(**options_overrides)
    loader = PolicyLoader(options)

    policies = loader.load_data(policy_config, 'config.json', validate=False)
    if policies:
        for p in policies:
            log.info("running policy %s", p.name)
            p.validate()
            p.push(event, context)
    return True
示例#26
0
 def get_context(self, config=None, session_factory=None, policy=None):
     if config is None:
         self.context_output_dir = self.get_temp_dir()
         config = Config.empty(output_dir=self.context_output_dir)
     ctx = ExecutionContext(session_factory, policy
                            or Bag({"name": "test-policy"}), config)
     return ctx
示例#27
0
def main():
    parser = setup_parser()
    options = parser.parse_args()

    log_level = logging.INFO
    if options.verbose:
        log_level = logging.DEBUG
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(logging.ERROR)
    logging.getLogger('c7n.cache').setLevel(logging.WARNING)

    if not options.regions:
        options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]

    files = []
    files.extend(itertools.chain(*options.config_files))
    files.extend(options.configs)
    options.config_files = files

    if not files:
        parser.print_help()
        sys.exit(1)

    policy_config = Config.empty(regions=options.regions,
                                 profile=options.profile,
                                 assume_role=options.assume_role)

    # use cloud provider to initialize policies to get region expansion
    policies = AWS().initialize_policies(load_policies(options, policy_config),
                                         policy_config)

    resources_gc_prefix(options, policy_config, policies)
示例#28
0
    def load_policy(
        self,
        data,
        config=None,
        session_factory=None,
        validate=C7N_VALIDATE,
        output_dir=None,
        cache=False,
    ):
        if validate:
            if not self.custodian_schema:
                self.custodian_schema = generate()
            errors = schema_validate({"policies": [data]},
                                     self.custodian_schema)
            if errors:
                raise errors[0]

        config = config or {}
        if not output_dir:
            temp_dir = self.get_temp_dir()
            config["output_dir"] = temp_dir
        if cache:
            config["cache"] = os.path.join(temp_dir, "c7n.cache")
            config["cache_period"] = 300
        conf = Config.empty(**config)
        p = policy.Policy(data, conf, session_factory)
        p.validate()
        return p
示例#29
0
def main():
    parser = setup_parser()
    argcomplete.autocomplete(parser)
    options = parser.parse_args()

    _setup_logger(options)

    # Support the deprecated -c option
    if getattr(options, 'config', None) is not None:
        options.configs.append(options.config)

    config = Config.empty(**vars(options))

    try:
        command = options.command
        if not callable(command):
            command = getattr(
                importlib.import_module(command.rsplit('.', 1)[0]),
                command.rsplit('.', 1)[-1])

        # Set the process name to something cleaner
        process_name = [os.path.basename(sys.argv[0])]
        process_name.extend(sys.argv[1:])
        setproctitle(' '.join(process_name))
        command(config)
    except Exception:
        if not options.debug:
            raise
        traceback.print_exc()
        pdb.post_mortem(sys.exc_info()[-1])
示例#30
0
    def test_resource_shadow_source_augment(self):
        shadowed = []
        bad = []
        cfg = Config.empty()

        for k, v in manager.resources.items():
            if not getattr(v.resource_type, "config_type", None):
                continue

            p = Bag({
                "name": "permcheck",
                "resource": k,
                'provider_name': 'aws'
            })
            ctx = self.get_context(config=cfg, policy=p)
            mgr = v(ctx, p)

            source = mgr.get_source("config")
            if not isinstance(source, ConfigSource):
                bad.append(k)

            if v.__dict__.get("augment"):
                shadowed.append(k)

        if shadowed:
            self.fail("%s have resource managers shadowing source augments" %
                      (", ".join(shadowed)))

        if bad:
            self.fail("%s have config types but no config source" %
                      (", ".join(bad)))
示例#31
0
def run(event, context):

    # policies file should always be valid in functions so do loading naively
    with open(context['config_file']) as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        log.error('Invalid policy config')
        return False

    options_overrides = \
        policy_config['policies'][0].get('mode', {}).get('execution-options', {})

    # setup our auth file location on disk
    options_overrides['authorization_file'] = context['auth_file']

    # if output_dir specified use that, otherwise make a temp directory
    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = get_tmp_output_dir()

    # merge all our options in
    options = Config.empty(**options_overrides)

    load_resources()

    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            p.push(event, context)
    return True
示例#32
0
class HandlerTest(BaseTest):
    @patch('c7n_azure.provider.Azure.initialize', return_value=Config.empty())
    @patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
           return_value=None)
    @patch('c7n.policy.Policy.push')
    def test_run(self, push_mock, _1, initialize_mock):
        context = {
            'config_file': join(dirname(__file__), 'data', 'test_config.json'),
            'auth_file': join(dirname(__file__), 'data', 'test_auth_file.json')
        }

        self.assertTrue(run(None, context, CUSTOM_SUBSCRIPTION_ID))

        push_mock.assert_called_once()
        self.assertEqual(push_mock.call_args_list[0], call(None, context))

        initialize_mock.assert_called_once()
        self.assertEqual(initialize_mock.call_args_list[0][0][0]['account_id'],
                         CUSTOM_SUBSCRIPTION_ID)
        self.assertEqual(
            initialize_mock.call_args_list[0][0][0]['authorization_file'],
            context['auth_file'])
        self.assertEqual(
            initialize_mock.call_args_list[0][0][0]['test_option'],
            "test_value")

    def test_run_empty_policy(self):
        context = {
            'config_file':
            join(dirname(__file__), 'data', 'test_config_empty.json'),
            'auth_file':
            join(dirname(__file__), 'data', 'test_auth_file.json')
        }

        self.assertFalse(run(None, context))
示例#33
0
def init_config(policy_config):
    """Get policy lambda execution configuration.

    cli parameters are serialized into the policy lambda config,
    we merge those with any policy specific execution options.

    --assume role and -s output directory get special handling, as
    to disambiguate any cli context.

    account id is sourced from the config options or from api call
    and cached as a global.

    Todo: this should get refactored out to mu.py as part of the
    write out of configuration, instead of runtime processed.
    """
    exec_options = policy_config.get('execution-options', {})

    # Remove some configuration options that don't make sense to translate from
    # cli to lambda automatically.
    #  - assume role on cli doesn't translate, it is the default lambda role and
    #    used to provision the lambda.
    #  - profile doesnt translate to lambda its `home` dir setup dependent
    #  - dryrun doesn't translate (and shouldn't be present)
    #  - region doesn't translate from cli (the lambda is bound to a region), and
    #    on the cli represents the region the lambda is provisioned in.
    for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
        exec_options.pop(k, None)

    # a cli local directory doesn't translate to lambda
    if not exec_options.get('output_dir', '').startswith('s3'):
        exec_options['output_dir'] = '/tmp'

    account_id = None
    # we can source account id from the cli parameters to avoid the sts call
    if exec_options.get('account_id'):
        account_id = exec_options['account_id']

    # merge with policy specific configuration
    exec_options.update(
        policy_config['policies'][0].get('mode', {}).get('execution-options', {}))

    # if using assume role in lambda ensure that the correct
    # execution account is captured in options.
    if 'assume_role' in exec_options:
        account_id = exec_options['assume_role'].split(':')[4]
    elif account_id is None:
        session = local_session(boto3.Session)
        account_id = get_account_id_from_sts(session)
    exec_options['account_id'] = account_id

    # Historical compatibility with manually set execution options
    # previously this was a boolean, its now a string value with the
    # boolean flag triggering a string value of 'aws'
    if 'metrics_enabled' in exec_options \
       and isinstance(exec_options['metrics_enabled'], bool) \
       and exec_options['metrics_enabled']:
        exec_options['metrics_enabled'] = 'aws'

    return Config.empty(**exec_options)
示例#34
0
 def get_related(self, resources):
     ctx = ExecutionContext(local_session(Session), self.data, Config.empty())
     manager = self.factory(ctx, self.data)
     related = manager.source.get_resources(None)
     if self.data.get('op'):
         return [r['id'] for r in related if self.match(r)]
     else:
         return [r['id'] for r in related]
示例#35
0
 def get_related(self, resources):
     ctx = ExecutionContext(local_session(Session), self.data, Config.empty())
     manager = self.factory(ctx, self.data)
     related = manager.source.get_resources(None)
     if self.data.get('op'):
         return [r['id'] for r in related if self.match(r)]
     else:
         return [r['id'] for r in related]
示例#36
0
 def get_context(self, config=None, session_factory=None, policy=None):
     if config is None:
         self.context_output_dir = self.get_temp_dir()
         config = Config.empty(output_dir=self.context_output_dir)
     ctx = ExecutionContext(
         session_factory, policy or Bag({
             "name": "test-policy", "provider_name": "aws"}), config)
     return ctx
示例#37
0
 def _get_policy_config(self, **kw):
     config = kw
     if kw.get('output_dir') is None or config.get('cache'):
         config["output_dir"] = temp_dir = self.get_temp_dir()
     if config.get('cache'):
         config["cache"] = os.path.join(temp_dir, "c7n.cache")
         config["cache_period"] = 300
     return Config.empty(**config)
示例#38
0
def init_config(policy_config):
    """Get policy lambda execution configuration.

    cli parameters are serialized into the policy lambda config,
    we merge those with any policy specific execution options.

    --assume role and -s output directory get special handling, as
    to disambiguate any cli context.

    account id is sourced from the config options or from api call
    and cached as a global
    """
    global account_id

    exec_options = policy_config.get('execution-options', {})

    # Remove some configuration options that don't make sense to translate from
    # cli to lambda automatically.
    #  - assume role on cli doesn't translate, it is the default lambda role and
    #    used to provision the lambda.
    #  - profile doesnt translate to lambda its `home` dir setup dependent
    #  - dryrun doesn't translate (and shouldn't be present)
    #  - region doesn't translate from cli (the lambda is bound to a region), and
    #    on the cli represents the region the lambda is provisioned in.
    for k in ('assume_role', 'profile', 'region', 'dryrun', 'cache'):
        exec_options.pop(k, None)

    # a cli local directory doesn't translate to lambda
    if not exec_options.get('output_dir', '').startswith('s3'):
        exec_options['output_dir'] = get_local_output_dir()

    # we can source account id from the cli parameters to avoid the sts call
    if exec_options.get('account_id'):
        account_id = exec_options['account_id']

    # merge with policy specific configuration
    exec_options.update(
        policy_config['policies'][0].get('mode', {}).get('execution-options', {}))

    # if using assume role in lambda ensure that the correct
    # execution account is captured in options.
    if 'assume_role' in exec_options:
        account_id = exec_options['assume_role'].split(':')[4]
    elif account_id is None:
        session = boto3.Session()
        account_id = get_account_id_from_sts(session)
    exec_options['account_id'] = account_id

    # Historical compatibility with manually set execution options
    # previously this was a boolean, its now a string value with the
    # boolean flag triggering a string value of 'aws'
    if 'metrics_enabled' in exec_options \
       and isinstance(exec_options['metrics_enabled'], bool) \
       and exec_options['metrics_enabled']:
        exec_options['metrics_enabled'] = 'aws'

    return Config.empty(**exec_options)
示例#39
0
def get_session(role, session_name, profile):
    region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1')
    stats = ApiStats(Bag(), Config.empty())
    if role:
        s = assumed_session(role, session_name, region=region)
    else:
        s = SessionFactory(region, profile)()
    stats(s)
    return stats, s
示例#40
0
 def get_context(self, config=None, session_factory=None, policy=None):
     if config is None:
         self.context_output_dir = self.get_temp_dir()
         config = Config.empty(output_dir=self.context_output_dir)
     ctx = ExecutionContext(
         session_factory,
         policy or Bag({'name': 'test-policy'}),
         config)
     return ctx
示例#41
0
def validate(options):
    load_resources()
    if len(options.configs) < 1:
        log.error('no config files specified')
        sys.exit(1)

    used_policy_names = set()
    schm = schema.generate()
    errors = []

    for config_file in options.configs:
        config_file = os.path.expanduser(config_file)
        if not os.path.exists(config_file):
            raise ValueError("Invalid path for config %r" % config_file)

        options.dryrun = True
        fmt = config_file.rsplit('.', 1)[-1]
        with open(config_file) as fh:
            if fmt in ('yml', 'yaml'):
                data = yaml.safe_load(fh.read())
            elif fmt in ('json',):
                data = json.load(fh)
            else:
                log.error("The config file must end in .json, .yml or .yaml.")
                raise ValueError("The config file must end in .json, .yml or .yaml.")

        errors += schema.validate(data, schm)
        conf_policy_names = {
            p.get('name', 'unknown') for p in data.get('policies', ())}
        dupes = conf_policy_names.intersection(used_policy_names)
        if len(dupes) >= 1:
            errors.append(ValueError(
                "Only one policy with a given name allowed, duplicates: %s" % (
                    ", ".join(dupes)
                )
            ))
        used_policy_names = used_policy_names.union(conf_policy_names)
        if not errors:
            null_config = Config.empty(dryrun=True, account_id='na', region='na')
            for p in data.get('policies', ()):
                try:
                    policy = Policy(p, null_config, Bag())
                    policy.validate()
                except Exception as e:
                    msg = "Policy: %s is invalid: %s" % (
                        p.get('name', 'unknown'), e)
                    errors.append(msg)
        if not errors:
            log.info("Configuration valid: {}".format(config_file))
            continue

        log.error("Configuration invalid: {}".format(config_file))
        for e in errors:
            log.error("%s" % e)
    if errors:
        sys.exit(1)
示例#42
0
def cli(**kwargs):
    policy_config = Config.empty()
    policies = PolicyCollection([
        p for p in load_policies(kwargs['config'], policy_config)
        if p.provider_name == 'azure'
    ], policy_config)

    session = policies.policies[0].session_factory()
    web_client = session.client('azure.mgmt.web.WebSiteManagementClient')

    deployments = {}
    credentials_load_failed = 0
    not_functions_policy = 0
    for p in policies:
        if not p.is_lambda:
            not_functions_policy += 1
            continue
        try:
            params = AzureFunctionMode(p).get_function_app_params()
            creds = web_client.web_apps.list_publishing_credentials(
                params.function_app_resource_group_name,
                params.function_app_name).result()
            deployments[p.name] = {'scm_uri': creds.scm_uri, 'status': None}
            log.info('Retrieved deployment credentials for %s policy', p.name)
        except Exception:
            log.error(
                'Unable to retrieve deployment credentials for %s policy',
                p.name)
            credentials_load_failed += 1

    wait_for_remote_builds(deployments)

    success = 0
    fail = 0
    not_found = 0
    for name, params in deployments.items():
        if params['status'] != DeploymentStatus.Succeeded:
            log.info('%s: %s', name, params['status'])
            if params['status'] == DeploymentStatus.Failed:
                log.error('Build logs can be retrieved here: %s',
                          params['scm_uri'])
                fail += 1
            else:
                not_found += 1
        else:
            success += 1

    log.info(
        'Policies total: %i, unable to load credentials: %i, not Functions mode: %i',
        len(policies), credentials_load_failed, not_functions_policy)
    log.info(
        'Status not found can happen if Linux Consumption function was deployed'
        'more than 2 hours ago.')
    log.info(
        'Deployments complete. Success: %i, Fail: %i, Status not found: %i',
        success, fail, not_found)
示例#43
0
    def test_initialize_default_account_id(self, get_subscription_id_mock):
        options = Config.empty()
        azure = Azure()
        azure.initialize(options)

        self.assertEqual(options['account_id'], DEFAULT_SUBSCRIPTION_ID)

        session = azure.get_session_factory(options)()
        session._initialize_session()
        self.assertEqual(session.subscription_id, DEFAULT_SUBSCRIPTION_ID)
示例#44
0
    def test_local_session_region(self):
        policies = [
            self.load_policy(
                {'name': 'ec2', 'resource': 'ec2'},
                config=Config.empty(region="us-east-1")),
            self.load_policy(
                {'name': 'ec2', 'resource': 'ec2'},
                config=Config.empty(region='us-west-2'))]
        previous = None
        previous_region = None
        for p in policies:
            self.assertEqual(p.options.region, p.session_factory.region)
            session = utils.local_session(p.session_factory)
            self.assertNotEqual(session.region_name, previous_region)
            self.assertNotEqual(session, previous)
            previous = session
            previous_region = p.options.region

        self.assertEqual(utils.local_session(p.session_factory), previous)
示例#45
0
    def initialize_tree(self, tree):
        assert not self.policy_files

        for tree_ent in tree:
            fpath = tree_ent.name
            if not self.matcher(fpath):
                continue
            self.policy_files[fpath] = PolicyCollection.from_data(
                yaml.safe_load(self.repo.get(tree[fpath].id).data),
                Config.empty(), fpath)
示例#46
0
    def get_azure_output(self):
        output = AzureStorageOutput(
            ExecutionContext(
                None,
                Bag(name="xyz"),
                Config.empty(output_dir="azure://mystorage.blob.core.windows.net/logs"),
            )
        )
        self.addCleanup(shutil.rmtree, output.root_dir)

        return output
示例#47
0
    def test_initialize_custom_account_id(self, get_subscription_id_mock):
        sample_account_id = "00000000-5106-4743-99b0-c129bfa71a47"
        options = Config.empty()
        options['account_id'] = sample_account_id
        azure = Azure()
        azure.initialize(options)
        self.assertEqual(options['account_id'], sample_account_id)

        session = azure.get_session_factory(options)()
        session._initialize_session()
        self.assertEqual(session.subscription_id, sample_account_id)
示例#48
0
 def _policy_file_rev(self, f, commit):
     try:
         return self._validate_policies(
             PolicyCollection.from_data(
                 yaml.safe_load(self.repo.get(commit.tree[f].id).data),
                 Config.empty(), f))
     except Exception as e:
         log.warning(
             "invalid policy file %s @ %s %s %s \n error:%s",
             f, str(commit.id)[:6], commit_date(commit).isoformat(),
             commit.author.name, e)
         return PolicyCollection()
示例#49
0
def dispatch_event(event, context):

    global account_id
    if account_id is None:
        session = boto3.Session()
        account_id = get_account_id_from_sts(session)

    error = event.get('detail', {}).get('errorCode')
    if error:
        log.debug("Skipping failed operation: %s" % error)
        return

    event['debug'] = True
    if event['debug']:
        log.info("Processing event\n %s", format_event(event))

    # policies file should always be valid in lambda so do loading naively
    with open('config.json') as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        return False

    # Initialize output directory, we've seen occassional perm issues with
    # lambda on temp directory and changing unix execution users, so
    # use a per execution temp space.
    output_dir = os.environ.get(
        'C7N_OUTPUT_DIR',
        '/tmp/' + str(uuid.uuid4()))
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except OSError as error:
            log.warning("Unable to make output directory: {}".format(error))

    # TODO. This enshrines an assumption of a single policy per lambda.
    options_overrides = policy_config[
        'policies'][0].get('mode', {}).get('execution-options', {})
    options_overrides['account_id'] = account_id
    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = output_dir
    options = Config.empty(**options_overrides)

    load_resources()
    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            p.push(event, context)
    return True
示例#50
0
    def load_policy(
            self, data, config=None):
        errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
        if errors:
            raise errors[0]

        config = config or {}

        temp_dir = self.get_temp_dir()
        config['output_dir'] = temp_dir

        conf = Config.empty(**config)
        p = policy.Policy(data, conf, Session)
        p.validate()
        return p
示例#51
0
    def get_azure_output(self, custom_pyformat=None):
        output_dir = "azure://mystorage.blob.core.windows.net/logs"
        if custom_pyformat:
            output_dir = AzureStorageOutput.join(output_dir, custom_pyformat)

        output = AzureStorageOutput(
            ExecutionContext(
                None,
                Bag(name="xyz", provider_name='azure'),
                Config.empty(output_dir=output_dir)
            ),
            {'url': output_dir},
        )
        self.addCleanup(shutil.rmtree, output.root_dir)

        return output
示例#52
0
    def load_policy(
            self, data, config=None, session_factory=None,
            validate=C7N_VALIDATE, output_dir=None, cache=False):
        if validate:
            errors = schema_validate({'policies': [data]}, C7N_SCHEMA)
            if errors:
                raise errors[0]

        config = config or {}
        if not output_dir:
            temp_dir = self.get_temp_dir()
            config['output_dir'] = temp_dir
        if cache:
            config['cache'] = os.path.join(temp_dir, 'c7n.cache')
            config['cache_period'] = 300
        conf = Config.empty(**config)
        p = policy.Policy(data, conf, session_factory)
        p.validate()
        return p
示例#53
0
def main():
    parser = setup_parser()
    options = parser.parse_args()

    log_level = logging.INFO
    if options.verbose:
        log_level = logging.DEBUG
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(logging.ERROR)
    logging.getLogger('c7n.cache').setLevel(logging.WARNING)

    if not options.regions:
        options.regions = [os.environ.get('AWS_DEFAULT_REGION', 'us-east-1')]

    files = []
    files.extend(itertools.chain(*options.config_files))
    files.extend(options.configs)
    options.config_files = files

    if not files:
        parser.print_help()
        sys.exit(1)

    policy_config = Config.empty(
        regions=options.regions,
        profile=options.profile,
        assume_role=options.assume_role)

    # use cloud provider to initialize policies to get region expansion
    policies = AWS().initialize_policies(
        PolicyCollection([
            p for p in load_policies(
                options, policy_config)
            if p.provider_name == 'aws'],
            policy_config),
        policy_config)

    resources_gc_prefix(options, policy_config, policies)
示例#54
0
def run_account(account, region, policies_config, output_path,
                cache_period, cache_path, metrics, dryrun, debug):
    """Execute a set of policies on an account.
    """
    logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
    CONN_CACHE.session = None
    CONN_CACHE.time = None

    # allow users to specify interpolated output paths
    if '{' not in output_path:
        output_path = os.path.join(output_path, account['name'], region)

    cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))

    config = Config.empty(
        region=region, cache=cache_path,
        cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
        account_id=account['account_id'], metrics_enabled=metrics,
        log_group=None, profile=None, external_id=None)

    env_vars = account_tags(account)

    if account.get('role'):
        if isinstance(account['role'], six.string_types):
            config['assume_role'] = account['role']
            config['external_id'] = account.get('external_id')
        else:
            env_vars.update(
                _get_env_creds(get_session(account, 'custodian', region), region))

    elif account.get('profile'):
        config['profile'] = account['profile']

    policies = PolicyCollection.from_data(policies_config, config)
    policy_counts = {}
    success = True
    st = time.time()

    with environ(**env_vars):
        for p in policies:
            # Variable expansion and non schema validation (not optional)
            p.expand_variables(p.get_variables(account.get('vars', {})))
            p.validate()

            log.debug(
                "Running policy:%s account:%s region:%s",
                p.name, account['name'], region)
            try:
                resources = p.run()
                policy_counts[p.name] = resources and len(resources) or 0
                if not resources:
                    continue
                log.info(
                    "Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
                    account['name'], region, p.name, len(resources),
                    time.time() - st)
            except ClientError as e:
                success = False
                if e.response['Error']['Code'] == 'AccessDenied':
                    log.warning('Access denied account:%s region:%s',
                                account['name'], region)
                    return policy_counts, success
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                continue
            except Exception as e:
                success = False
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                if not debug:
                    continue
                import traceback, pdb, sys
                traceback.print_exc()
                pdb.post_mortem(sys.exc_info()[-1])
                raise

    return policy_counts, success
示例#55
0
def run_account(account, region, policies_config, output_path,
                cache_period, metrics, dryrun, debug):
    """Execute a set of policies on an account.
    """
    logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
    CONN_CACHE.session = None
    CONN_CACHE.time = None
    output_path = os.path.join(output_path, account['name'], region)
    if not os.path.exists(output_path):
        os.makedirs(output_path)

    cache_path = os.path.join(output_path, "c7n.cache")

    config = Config.empty(
        region=region,
        cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
        account_id=account['account_id'], metrics_enabled=metrics,
        cache=cache_path, log_group=None, profile=None, external_id=None)
    if account.get('role'):
        config['assume_role'] = account['role']
        config['external_id'] = account.get('external_id')
    elif account.get('profile'):
        config['profile'] = account['profile']

    policies = PolicyCollection.from_data(policies_config, config)
    policy_counts = {}
    st = time.time()
    with environ(**account_tags(account)):
        for p in policies:
            log.debug(
                "Running policy:%s account:%s region:%s",
                p.name, account['name'], region)
            try:
                resources = p.run()
                policy_counts[p.name] = resources and len(resources) or 0
                if not resources:
                    continue
                log.info(
                    "Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
                    account['name'], region, p.name, len(resources),
                    time.time() - st)
            except ClientError as e:
                if e.response['Error']['Code'] == 'AccessDenied':
                    log.warning('Access denied account:%s region:%s',
                                account['name'], region)
                    return policy_counts
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                continue
            except Exception as e:
                log.error(
                    "Exception running policy:%s account:%s region:%s error:%s",
                    p.name, account['name'], region, e)
                if not debug:
                    continue
                import traceback, pdb, sys
                traceback.print_exc()
                pdb.post_mortem(sys.exc_info()[-1])
                raise

    return policy_counts
示例#56
0
 def __init__(self, policies=None, options=None):
     self.policies = policies or []
     self.options = options or Config.empty()
     self.pmap = {p.name: p for p in self.policies}
示例#57
0
def dispatch_event(event, context):

    global account_id

    error = event.get('detail', {}).get('errorCode')
    if error:
        log.debug("Skipping failed operation: %s" % error)
        return

    event['debug'] = True
    if event['debug']:
        log.info("Processing event\n %s", format_event(event))

    # Policies file should always be valid in lambda so do loading naively
    with open('config.json') as f:
        policy_config = json.load(f)

    if not policy_config or not policy_config.get('policies'):
        return False

    # Initialize output directory, we've seen occassional perm issues with
    # lambda on temp directory and changing unix execution users, so
    # use a per execution temp space.
    output_dir = os.environ.get(
        'C7N_OUTPUT_DIR',
        '/tmp/' + str(uuid.uuid4()))
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except OSError as error:
            log.warning("Unable to make output directory: {}".format(error))

    # TODO. This enshrines an assumption of a single policy per lambda.
    options_overrides = policy_config[
        'policies'][0].get('mode', {}).get('execution-options', {})

    # if using assume role in lambda ensure that the correct
    # execution account is captured in options.
    if 'assume_role' in options_overrides:
        account_id = options_overrides['assume_role'].split(':')[4]
    elif account_id is None:
        session = boto3.Session()
        account_id = get_account_id_from_sts(session)

    # Historical compatibility with manually set execution options
    # previously this was a boolean, its now a string value with the
    # boolean flag triggering a string value of 'aws'
    if 'metrics_enabled' in options_overrides and isinstance(
            options_overrides['metrics_enabled'], bool):
        options_overrides['metrics_enabled'] = 'aws'

    options_overrides['account_id'] = account_id

    if 'output_dir' not in options_overrides:
        options_overrides['output_dir'] = output_dir
    options = Config.empty(**options_overrides)

    policies = PolicyCollection.from_data(policy_config, options)
    if policies:
        for p in policies:
            p.push(event, context)
    return True
示例#58
0
def report(config, output, use, output_dir, accounts,
           field, no_default_fields, tags, region, debug, verbose,
           policy, policy_tags, format, resource):
    """report on a cross account policy execution."""
    accounts_config, custodian_config, executor = init(
        config, use, debug, verbose, accounts, tags, policy,
        resource=resource, policy_tags=policy_tags)

    resource_types = set()
    for p in custodian_config.get('policies'):
        resource_types.add(p['resource'])
    if len(resource_types) > 1:
        raise ValueError("can only report on one resource type at a time")
    elif not len(custodian_config['policies']) > 0:
        raise ValueError("no matching policies found")

    records = []
    with executor(max_workers=WORKER_COUNT) as w:
        futures = {}
        for a in accounts_config.get('accounts', ()):
            for r in resolve_regions(region or a.get('regions', ())):
                futures[w.submit(
                    report_account,
                    a, r,
                    custodian_config,
                    output_dir,
                    debug)] = (a, r)

        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                if debug:
                    raise
                log.warning(
                    "Error running policy in %s @ %s exception: %s",
                    a['name'], r, f.exception())
            records.extend(f.result())

    log.debug(
        "Found %d records across %d accounts and %d policies",
        len(records), len(accounts_config['accounts']),
        len(custodian_config['policies']))

    if format == 'json':
        dumps(records, output, indent=2)
        return

    prefix_fields = OrderedDict(
        (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
    config = Config.empty()
    factory = resource_registry.get(list(resource_types)[0])

    formatter = Formatter(
        factory.resource_type,
        extra_fields=field,
        include_default_fields=not(no_default_fields),
        include_region=False,
        include_policy=False,
        fields=prefix_fields)

    rows = formatter.to_csv(records, unique=False)
    writer = UnicodeWriter(output, formatter.headers())
    writer.writerow(formatter.headers())
    writer.writerows(rows)