def validate(options):
load_resources()
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
schm = schema.generate()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml'):
data = yaml.safe_load(fh.read())
elif fmt in ('json',):
data = json.load(fh)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError("The config file must end in .json, .yml or .yaml.")
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown') for p in data.get('policies', ())}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(ValueError(
"Only one policy with a given name allowed, duplicates: %s" % (
", ".join(dupes)
)
))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True, account_id='na', region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (
p.get('name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
class Missing(Filter):
"""Assert the absence of a particular resource.
Intended for use at a logical account/subscription/project level
This works as an effectively an embedded policy thats evaluated.
"""
schema = type_schema(
'missing', policy={'type': 'object'},
required=['policy'])
def __init__(self, data, manager):
super(Missing, self).__init__(data, manager)
self.data['policy']['name'] = self.manager.ctx.policy.name
self.embedded_policy = Policy(
self.data['policy'], self.manager.config, self.manager.session_factory)
def validate(self):
if 'mode' in self.data['policy']:
raise PolicyValidationError(
"Execution mode can't be specified in "
"embedded policy %s" % self.data)
if 'actions' in self.data['policy']:
raise PolicyValidationError(
"Actions can't be specified in "
"embedded policy %s" % self.data)
self.embedded_policy.validate()
return self
def get_permissions(self):
return self.embedded_policy.get_permissions()
def process(self, resources, event=None):
provider = clouds[self.manager.ctx.policy.provider_name]()
# if the embedded policy only specifies one region, or only
# being executed on a single region.
if self.embedded_policy.region or len(self.manager.config.regions) <= 1:
if (self.embedded_policy.region and
self.embedded_policy.region != self.manager.config.region):
return []
self.embedded_policy.expand_variables(self.embedded_policy.get_variables())
return not self.embedded_policy.poll() and resources or []
# For regional resources and multi-region execution, the policy matches if
# the resource is missing in any region.
found = {}
for p in provider.initialize_policies(
PolicyCollection([self.embedded_policy], self.manager.config),
self.manager.config):
p.expand_variables(p.get_variables())
p.validate()
found[p.options.region] = p.poll()
if not all(found.values()):
return resources
return []
def initialize_policies(self, policy_collection, options):
"""Return a set of policies targetted to the given regions.
Supports symbolic regions like 'all'. This will automatically
filter out policies if they are being targetted to a region that
does not support the service. Global services will target a
single region (us-east-1 if only all specified, else first
region in the list).
Note for region partitions (govcloud and china) an explicit
region from the partition must be passed in.
"""
from c7n.policy import Policy, PolicyCollection
policies = []
service_region_map, resource_service_map = get_service_region_map(
options.regions, policy_collection.resource_types)
if 'all' in options.regions:
enabled_regions = {
r['RegionName']
for r in get_profile_session(options).client(
'ec2').describe_regions(
Filters=[{
'Name': 'opt-in-status',
'Values': ['opt-in-not-required', 'opted-in']
}]).get('Regions')
}
for p in policy_collection:
if 'aws.' in p.resource_type:
_, resource_type = p.resource_type.split('.', 1)
else:
resource_type = p.resource_type
available_regions = service_region_map.get(
resource_service_map.get(resource_type), ())
# its a global service/endpoint, use user provided region
# or us-east-1.
if not available_regions and options.regions:
candidates = [r for r in options.regions if r != 'all']
candidate = candidates and candidates[0] or 'us-east-1'
svc_regions = [candidate]
elif 'all' in options.regions:
svc_regions = list(
set(available_regions).intersection(enabled_regions))
else:
svc_regions = options.regions
for region in svc_regions:
if available_regions and region not in available_regions:
level = ('all' in options.regions and logging.DEBUG
or logging.WARNING)
# TODO: fixme
policy_collection.log.log(
level,
"policy:%s resources:%s not available in region:%s",
p.name, p.resource_type, region)
continue
options_copy = copy.copy(options)
options_copy.region = str(region)
if len(options.regions
) > 1 or 'all' in options.regions and getattr(
options, 'output_dir', None):
options_copy.output_dir = join_output(
options.output_dir, region)
policies.append(
Policy(
p.data,
options_copy,
session_factory=policy_collection.session_factory()))
return PolicyCollection(
# order policies by region to minimize local session invalidation.
# note relative ordering of policies must be preserved, python sort
# is stable.
sorted(policies, key=operator.attrgetter('options.region')),
options)
def test_cwe_update_config_and_code(self):
# Originally this was testing the no update case.. but
# That is tricky to record, any updates to the code end up
# causing issues due to checksum mismatches which imply updating
# the function code / which invalidate the recorded data and
# the focus of the test.
session_factory = self.replay_flight_data('test_cwe_update',
zdata=True)
p = Policy(
{
'resource':
's3',
'name':
's3-bucket-policy',
'mode': {
'type': 'cloudtrail',
'events': ["CreateBucket"],
},
'filters': [{
'type': 'missing-policy-statement',
'statement_ids': ['RequireEncryptedPutObject']
}],
'actions': ['no-op']
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, 'Dev', role=self.role)
self.addCleanup(mgr.remove, pl)
p = Policy(
{
'resource':
's3',
'name':
's3-bucket-policy',
'mode': {
'type':
'cloudtrail',
'memory':
256,
'events': [
"CreateBucket", {
'event': 'PutBucketPolicy',
'ids': 'requestParameters.bucketName',
'source': 's3.amazonaws.com'
}
]
},
'filters': [{
'type': 'missing-policy-statement',
'statement_ids': ['RequireEncryptedPutObject']
}],
'actions': ['no-op']
}, Config.empty())
output = self.capture_logging('custodian.lambda', level=logging.DEBUG)
result2 = mgr.publish(PolicyLambda(p), 'Dev', role=self.role)
lines = output.getvalue().strip().split('\n')
self.assertTrue(
'Updating function custodian-s3-bucket-policy code' in lines)
self.assertTrue(
'Updating function: custodian-s3-bucket-policy config' in lines)
self.assertEqual(result['FunctionName'], result2['FunctionName'])
# drive by coverage
functions = [
i for i in mgr.list_functions()
if i['FunctionName'] == 'custodian-s3-bucket-policy'
]
self.assertTrue(len(functions), 1)
start = 0L
end = long(time.time() * 1000)
self.assertEqual(list(mgr.logs(pl, start, end)), [])
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals
import unittest
from dateutil.parser import parse as date_parse
from c7n.policy import Policy
from c7n.reports.csvout import Formatter
from .common import Config, load_data
EC2_POLICY = Policy(
{
'name': 'report-test-ec2',
'resource': 'ec2',
},
Config.empty(),
)
ASG_POLICY = Policy(
{
'name': 'report-test-asg',
'resource': 'asg',
},
Config.empty(),
)
ELB_POLICY = Policy(
{
'name': 'report-test-elb',
'resource': 'elb',
},
def test_cwe_update_config_and_code(self):
# Originally this was testing the no update case.. but
# That is tricky to record, any updates to the code end up
# causing issues due to checksum mismatches which imply updating
# the function code / which invalidate the recorded data and
# the focus of the test.
session_factory = self.replay_flight_data("test_cwe_update", zdata=True)
p = Policy(
{
"resource": "s3",
"name": "s3-bucket-policy",
"mode": {"type": "cloudtrail", "events": ["CreateBucket"], 'runtime': 'python2.7'},
"filters": [
{
"type": "missing-policy-statement",
"statement_ids": ["RequireEncryptedPutObject"],
}
],
"actions": ["no-op"],
},
Config.empty(),
)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, "Dev", role=ROLE)
self.addCleanup(mgr.remove, pl)
p = Policy(
{
"resource": "s3",
"name": "s3-bucket-policy",
"mode": {
"type": "cloudtrail",
"memory": 256,
'runtime': 'python2.7',
"events": [
"CreateBucket",
{
"event": "PutBucketPolicy",
"ids": "requestParameters.bucketName",
"source": "s3.amazonaws.com",
},
],
},
"filters": [
{
"type": "missing-policy-statement",
"statement_ids": ["RequireEncryptedPutObject"],
}
],
"actions": ["no-op"],
},
Config.empty(),
)
output = self.capture_logging("custodian.serverless", level=logging.DEBUG)
result2 = mgr.publish(PolicyLambda(p), "Dev", role=ROLE)
lines = output.getvalue().strip().split("\n")
self.assertTrue("Updating function custodian-s3-bucket-policy code" in lines)
self.assertTrue(
"Updating function: custodian-s3-bucket-policy config MemorySize" in lines)
self.assertEqual(result["FunctionName"], result2["FunctionName"])
# drive by coverage
functions = [
i
for i in mgr.list_functions()
if i["FunctionName"] == "custodian-s3-bucket-policy"
]
self.assertTrue(len(functions), 1)
start = 0
end = time.time() * 1000
self.assertEqual(list(mgr.logs(pl, start, end)), [])
def validate(options):
from c7n import schema
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
structure = StructureParser()
errors = []
found_deprecations = False
footnotes = deprecated.Footnotes()
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml', 'json'):
# our loader is safe loader derived.
data = yaml.load(
fh.read(),
Loader=DuplicateKeyCheckLoader) # nosec nosemgrep
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError(
"The config file must end in .json, .yml or .yaml.")
try:
structure.validate(data)
except PolicyValidationError as e:
log.error("Configuration invalid: {}".format(config_file))
log.error("%s" % e)
errors.append(e)
continue
load_resources(structure.get_resource_types(data))
schm = schema.generate()
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown')
for p in data.get('policies', ())
}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(
ValueError(
"Only one policy with a given name allowed, duplicates: %s"
% (", ".join(dupes))))
used_policy_names = used_policy_names.union(conf_policy_names)
source_locator = None
if fmt in ('yml', 'yaml'):
# For yaml files there is at least the expectation that the policy
# name is on a line by itself. With JSON, the file could be one big
# line. At this stage we are only attempting to find line number for
# policies in yaml files.
source_locator = SourceLocator(config_file)
if not errors:
null_config = Config.empty(dryrun=True,
account_id='na',
region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
# If the policy is invalid, there isn't much point checking
# for deprecated usage as there is no guarantee as to the
# state of the policy.
if options.check_deprecations != deprecated.SKIP:
report = deprecated.report(policy)
if report:
found_deprecations = True
log.warning(
"deprecated usage found in policy\n" +
report.format(source_locator=source_locator,
footnotes=footnotes))
except Exception as e:
msg = "Policy: %s is invalid: %s" % (p.get(
'name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if found_deprecations:
notes = footnotes()
if notes:
log.warning("deprecation footnotes:\n" + notes)
if options.check_deprecations == deprecated.STRICT:
sys.exit(1)
if errors:
sys.exit(1)
def validate(options):
from c7n import schema
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
structure = StructureParser()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml', 'json'):
data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError(
"The config file must end in .json, .yml or .yaml.")
try:
structure.validate(data)
except PolicyValidationError as e:
log.error("Configuration invalid: {}".format(config_file))
log.error("%s" % e)
errors.append(e)
continue
load_resources(structure.get_resource_types(data))
schm = schema.generate()
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown')
for p in data.get('policies', ())
}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(
ValueError(
"Only one policy with a given name allowed, duplicates: %s"
% (", ".join(dupes))))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True,
account_id='na',
region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (p.get(
'name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals
import unittest
from c7n.policy import Policy
from c7n.reports.csvout import Formatter
from .common import Config, load_data
EC2_POLICY = Policy({
"name": "report-test-ec2",
"resource": "ec2"
}, Config.empty())
ASG_POLICY = Policy({
"name": "report-test-asg",
"resource": "asg"
}, Config.empty())
ELB_POLICY = Policy({
"name": "report-test-elb",
"resource": "elb"
}, Config.empty())
class TestEC2Report(unittest.TestCase):
def setUp(self):
data = load_data("report.json")
self.records = data["ec2"]["records"]
def __init__(self, data, manager):
super(Missing, self).__init__(data, manager)
self.data['policy']['name'] = self.manager.ctx.policy.name
self.embedded_policy = Policy(self.data['policy'], self.manager.config,
self.manager.session_factory)
def __init__(self, data, manager):
super(Missing, self).__init__(data, manager)
self.data['policy']['name'] = self.manager.ctx.policy.name
self.embedded_policy = Policy(
self.data['policy'], self.manager.config, self.manager.session_factory)
def validate(options):
load_resources()
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
schm = schema.generate()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml'):
data = yaml.safe_load(fh.read())
elif fmt in ('json', ):
data = json.load(fh)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError(
"The config file must end in .json, .yml or .yaml.")
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown')
for p in data.get('policies', ())
}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(
ValueError(
"Only one policy with a given name allowed, duplicates: %s"
% (", ".join(dupes))))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Bag(dryrun=True,
log_group=None,
cache=None,
assume_role="na")
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (p.get(
'name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
