def validate(options): load_resources() if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() schm = schema.generate() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml'): data = yaml.safe_load(fh.read()) elif fmt in ('json',): data = json.load(fh) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError("The config file must end in .json, .yml or .yaml.") errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ())} dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append(ValueError( "Only one policy with a given name allowed, duplicates: %s" % ( ", ".join(dupes) ) )) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % ( p.get('name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
class Missing(Filter): """Assert the absence of a particular resource. Intended for use at a logical account/subscription/project level This works as an effectively an embedded policy thats evaluated. """ schema = type_schema( 'missing', policy={'type': 'object'}, required=['policy']) def __init__(self, data, manager): super(Missing, self).__init__(data, manager) self.data['policy']['name'] = self.manager.ctx.policy.name self.embedded_policy = Policy( self.data['policy'], self.manager.config, self.manager.session_factory) def validate(self): if 'mode' in self.data['policy']: raise PolicyValidationError( "Execution mode can't be specified in " "embedded policy %s" % self.data) if 'actions' in self.data['policy']: raise PolicyValidationError( "Actions can't be specified in " "embedded policy %s" % self.data) self.embedded_policy.validate() return self def get_permissions(self): return self.embedded_policy.get_permissions() def process(self, resources, event=None): provider = clouds[self.manager.ctx.policy.provider_name]() # if the embedded policy only specifies one region, or only # being executed on a single region. if self.embedded_policy.region or len(self.manager.config.regions) <= 1: if (self.embedded_policy.region and self.embedded_policy.region != self.manager.config.region): return [] self.embedded_policy.expand_variables(self.embedded_policy.get_variables()) return not self.embedded_policy.poll() and resources or [] # For regional resources and multi-region execution, the policy matches if # the resource is missing in any region. found = {} for p in provider.initialize_policies( PolicyCollection([self.embedded_policy], self.manager.config), self.manager.config): p.expand_variables(p.get_variables()) p.validate() found[p.options.region] = p.poll() if not all(found.values()): return resources return []
def initialize_policies(self, policy_collection, options): """Return a set of policies targetted to the given regions. Supports symbolic regions like 'all'. This will automatically filter out policies if they are being targetted to a region that does not support the service. Global services will target a single region (us-east-1 if only all specified, else first region in the list). Note for region partitions (govcloud and china) an explicit region from the partition must be passed in. """ from c7n.policy import Policy, PolicyCollection policies = [] service_region_map, resource_service_map = get_service_region_map( options.regions, policy_collection.resource_types) if 'all' in options.regions: enabled_regions = { r['RegionName'] for r in get_profile_session(options).client( 'ec2').describe_regions( Filters=[{ 'Name': 'opt-in-status', 'Values': ['opt-in-not-required', 'opted-in'] }]).get('Regions') } for p in policy_collection: if 'aws.' in p.resource_type: _, resource_type = p.resource_type.split('.', 1) else: resource_type = p.resource_type available_regions = service_region_map.get( resource_service_map.get(resource_type), ()) # its a global service/endpoint, use user provided region # or us-east-1. if not available_regions and options.regions: candidates = [r for r in options.regions if r != 'all'] candidate = candidates and candidates[0] or 'us-east-1' svc_regions = [candidate] elif 'all' in options.regions: svc_regions = list( set(available_regions).intersection(enabled_regions)) else: svc_regions = options.regions for region in svc_regions: if available_regions and region not in available_regions: level = ('all' in options.regions and logging.DEBUG or logging.WARNING) # TODO: fixme policy_collection.log.log( level, "policy:%s resources:%s not available in region:%s", p.name, p.resource_type, region) continue options_copy = copy.copy(options) options_copy.region = str(region) if len(options.regions ) > 1 or 'all' in options.regions and getattr( options, 'output_dir', None): options_copy.output_dir = join_output( options.output_dir, region) policies.append( Policy( p.data, options_copy, session_factory=policy_collection.session_factory())) return PolicyCollection( # order policies by region to minimize local session invalidation. # note relative ordering of policies must be preserved, python sort # is stable. sorted(policies, key=operator.attrgetter('options.region')), options)
def test_cwe_update_config_and_code(self): # Originally this was testing the no update case.. but # That is tricky to record, any updates to the code end up # causing issues due to checksum mismatches which imply updating # the function code / which invalidate the recorded data and # the focus of the test. session_factory = self.replay_flight_data('test_cwe_update', zdata=True) p = Policy( { 'resource': 's3', 'name': 's3-bucket-policy', 'mode': { 'type': 'cloudtrail', 'events': ["CreateBucket"], }, 'filters': [{ 'type': 'missing-policy-statement', 'statement_ids': ['RequireEncryptedPutObject'] }], 'actions': ['no-op'] }, Config.empty()) pl = PolicyLambda(p) mgr = LambdaManager(session_factory) result = mgr.publish(pl, 'Dev', role=self.role) self.addCleanup(mgr.remove, pl) p = Policy( { 'resource': 's3', 'name': 's3-bucket-policy', 'mode': { 'type': 'cloudtrail', 'memory': 256, 'events': [ "CreateBucket", { 'event': 'PutBucketPolicy', 'ids': 'requestParameters.bucketName', 'source': 's3.amazonaws.com' } ] }, 'filters': [{ 'type': 'missing-policy-statement', 'statement_ids': ['RequireEncryptedPutObject'] }], 'actions': ['no-op'] }, Config.empty()) output = self.capture_logging('custodian.lambda', level=logging.DEBUG) result2 = mgr.publish(PolicyLambda(p), 'Dev', role=self.role) lines = output.getvalue().strip().split('\n') self.assertTrue( 'Updating function custodian-s3-bucket-policy code' in lines) self.assertTrue( 'Updating function: custodian-s3-bucket-policy config' in lines) self.assertEqual(result['FunctionName'], result2['FunctionName']) # drive by coverage functions = [ i for i in mgr.list_functions() if i['FunctionName'] == 'custodian-s3-bucket-policy' ] self.assertTrue(len(functions), 1) start = 0L end = long(time.time() * 1000) self.assertEqual(list(mgr.logs(pl, start, end)), [])
# See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, division, print_function, unicode_literals import unittest from dateutil.parser import parse as date_parse from c7n.policy import Policy from c7n.reports.csvout import Formatter from .common import Config, load_data EC2_POLICY = Policy( { 'name': 'report-test-ec2', 'resource': 'ec2', }, Config.empty(), ) ASG_POLICY = Policy( { 'name': 'report-test-asg', 'resource': 'asg', }, Config.empty(), ) ELB_POLICY = Policy( { 'name': 'report-test-elb', 'resource': 'elb', },
def test_cwe_update_config_and_code(self): # Originally this was testing the no update case.. but # That is tricky to record, any updates to the code end up # causing issues due to checksum mismatches which imply updating # the function code / which invalidate the recorded data and # the focus of the test. session_factory = self.replay_flight_data("test_cwe_update", zdata=True) p = Policy( { "resource": "s3", "name": "s3-bucket-policy", "mode": {"type": "cloudtrail", "events": ["CreateBucket"], 'runtime': 'python2.7'}, "filters": [ { "type": "missing-policy-statement", "statement_ids": ["RequireEncryptedPutObject"], } ], "actions": ["no-op"], }, Config.empty(), ) pl = PolicyLambda(p) mgr = LambdaManager(session_factory) result = mgr.publish(pl, "Dev", role=ROLE) self.addCleanup(mgr.remove, pl) p = Policy( { "resource": "s3", "name": "s3-bucket-policy", "mode": { "type": "cloudtrail", "memory": 256, 'runtime': 'python2.7', "events": [ "CreateBucket", { "event": "PutBucketPolicy", "ids": "requestParameters.bucketName", "source": "s3.amazonaws.com", }, ], }, "filters": [ { "type": "missing-policy-statement", "statement_ids": ["RequireEncryptedPutObject"], } ], "actions": ["no-op"], }, Config.empty(), ) output = self.capture_logging("custodian.serverless", level=logging.DEBUG) result2 = mgr.publish(PolicyLambda(p), "Dev", role=ROLE) lines = output.getvalue().strip().split("\n") self.assertTrue("Updating function custodian-s3-bucket-policy code" in lines) self.assertTrue( "Updating function: custodian-s3-bucket-policy config MemorySize" in lines) self.assertEqual(result["FunctionName"], result2["FunctionName"]) # drive by coverage functions = [ i for i in mgr.list_functions() if i["FunctionName"] == "custodian-s3-bucket-policy" ] self.assertTrue(len(functions), 1) start = 0 end = time.time() * 1000 self.assertEqual(list(mgr.logs(pl, start, end)), [])
def validate(options): from c7n import schema if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() structure = StructureParser() errors = [] found_deprecations = False footnotes = deprecated.Footnotes() for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml', 'json'): # our loader is safe loader derived. data = yaml.load( fh.read(), Loader=DuplicateKeyCheckLoader) # nosec nosemgrep else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError( "The config file must end in .json, .yml or .yaml.") try: structure.validate(data) except PolicyValidationError as e: log.error("Configuration invalid: {}".format(config_file)) log.error("%s" % e) errors.append(e) continue load_resources(structure.get_resource_types(data)) schm = schema.generate() errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ()) } dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append( ValueError( "Only one policy with a given name allowed, duplicates: %s" % (", ".join(dupes)))) used_policy_names = used_policy_names.union(conf_policy_names) source_locator = None if fmt in ('yml', 'yaml'): # For yaml files there is at least the expectation that the policy # name is on a line by itself. With JSON, the file could be one big # line. At this stage we are only attempting to find line number for # policies in yaml files. source_locator = SourceLocator(config_file) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() # If the policy is invalid, there isn't much point checking # for deprecated usage as there is no guarantee as to the # state of the policy. if options.check_deprecations != deprecated.SKIP: report = deprecated.report(policy) if report: found_deprecations = True log.warning( "deprecated usage found in policy\n" + report.format(source_locator=source_locator, footnotes=footnotes)) except Exception as e: msg = "Policy: %s is invalid: %s" % (p.get( 'name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if found_deprecations: notes = footnotes() if notes: log.warning("deprecation footnotes:\n" + notes) if options.check_deprecations == deprecated.STRICT: sys.exit(1) if errors: sys.exit(1)
def validate(options): from c7n import schema if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() structure = StructureParser() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml', 'json'): data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError( "The config file must end in .json, .yml or .yaml.") try: structure.validate(data) except PolicyValidationError as e: log.error("Configuration invalid: {}".format(config_file)) log.error("%s" % e) errors.append(e) continue load_resources(structure.get_resource_types(data)) schm = schema.generate() errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ()) } dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append( ValueError( "Only one policy with a given name allowed, duplicates: %s" % (", ".join(dupes)))) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % (p.get( 'name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, division, print_function, unicode_literals import unittest from c7n.policy import Policy from c7n.reports.csvout import Formatter from .common import Config, load_data EC2_POLICY = Policy({ "name": "report-test-ec2", "resource": "ec2" }, Config.empty()) ASG_POLICY = Policy({ "name": "report-test-asg", "resource": "asg" }, Config.empty()) ELB_POLICY = Policy({ "name": "report-test-elb", "resource": "elb" }, Config.empty()) class TestEC2Report(unittest.TestCase): def setUp(self): data = load_data("report.json") self.records = data["ec2"]["records"]
def __init__(self, data, manager): super(Missing, self).__init__(data, manager) self.data['policy']['name'] = self.manager.ctx.policy.name self.embedded_policy = Policy(self.data['policy'], self.manager.config, self.manager.session_factory)
def __init__(self, data, manager): super(Missing, self).__init__(data, manager) self.data['policy']['name'] = self.manager.ctx.policy.name self.embedded_policy = Policy( self.data['policy'], self.manager.config, self.manager.session_factory)
def validate(options): load_resources() if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() schm = schema.generate() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml'): data = yaml.safe_load(fh.read()) elif fmt in ('json', ): data = json.load(fh) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError( "The config file must end in .json, .yml or .yaml.") errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ()) } dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append( ValueError( "Only one policy with a given name allowed, duplicates: %s" % (", ".join(dupes)))) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Bag(dryrun=True, log_group=None, cache=None, assume_role="na") for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % (p.get( 'name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)