def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy, resource=resource)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
records = []
with executor(max_workers=16) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
account_regions = region or a['regions']
for r in account_regions:
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Bag.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=True,
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = csv.writer(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger('botocore').setLevel(logging.WARNING)
parser = setup_parser()
options = parser.parse_args()
options.log_group = None
options.cache = None
factory = SessionFactory(
options.region, options.profile, options.assume_role)
session = factory()
client = session.client('cloudwatch')
load_resources()
policies = load(options, options.config)
if options.start and options.end:
start = options.start
end = options.end
elif options.days:
end = datetime.utcnow()
start = end - timedelta(options.days)
data = {}
for p in policies:
logging.info('Getting %s metrics', p)
data[p.name] = p.get_metrics(start, end, options.period)
print dumps(data, indent=2)
def report(policy, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
formatter = Formatter(
policy.resource_manager,
extra_fields=options.field,
no_default_fields=options.no_default_fields,
)
if policy.ctx.output.use_s3():
records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
records = fs_record_set(policy.ctx.output_path, policy.name)
log.debug("Found %d records", len(records))
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def poll(self):
"""Query resources and apply policy."""
with self.ctx:
self.log.info("Running policy %s" % self.name)
s = time.time()
resources = self.resource_manager.resources()
rt = time.time() - s
self.log.info(
"policy: %s resource:%s has count:%d time:%0.2f" % (
self.name, self.resource_type, len(resources), rt))
self.ctx.metrics.put_metric(
"ResourceCount", len(resources), "Count", Scope="Policy")
self.ctx.metrics.put_metric(
"ResourceTime", rt, "Seconds", Scope="Policy")
self._write_file(
'resources.json', utils.dumps(resources, indent=2))
if not resources:
return []
if self.options.dryrun and not self.resource_manager.supports_dry_run:
self.log.debug("dryrun: skipping actions")
return resources
at = time.time()
for a in self.resource_manager.actions:
s = time.time()
results = a.process(resources)
self.log.info(
"policy: %s action: %s resources: %d execution_time: %0.2f" % (
self.name, a.name, len(resources), time.time()-s))
self._write_file("action-%s" % a.name, utils.dumps(results))
self.ctx.metrics.put_metric(
"ActionTime", time.time() - at, "Seconds", Scope="Policy")
return resources
def report(policy, start_date, options, output_fh, raw_output_fh=None, filters=None):
"""Format a policy's extant records into a report."""
formatter_cls = RECORD_TYPE_FORMATTERS.get(policy.resource_type)
formatter = formatter_cls(
extra_fields=options.field, no_default_fields=options.no_default_fields)
if formatter is None:
raise ValueError(
"No formatter defined for resource type '%s', valid options: %s" % (
policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS)))
if policy.ctx.output.use_s3():
records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
records = fs_record_set(policy.ctx.output_path, policy.name)
rows = formatter.to_csv(records)
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(policy, start_date, output_fh, raw_output_fh=None, filters=None):
"""Format a policy's extant records into a report."""
formatter = RECORD_TYPE_FORMATTERS.get(policy.resource_type)
if formatter is None:
raise ValueError(
"No formatter for resource type %s, valid: %s" % (
policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS)))
if policy.ctx.output_path.startswith('s3'):
records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
records = fs_record_set(policy.ctx.output_path, policy.name)
rows = formatter.to_csv(records)
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(policy, start_date, options, output_fh, raw_output_fh=None, filters=None):
"""Format a policy's extant records into a report."""
if not policy.resource_manager.report_fields:
raise ValueError(
"No formatter configured for resource type '%s', valid options: %s" % (
policy.resource_type))
formatter = Formatter(
policy.resource_manager,
extra_fields=options.field,
no_default_fields=options.no_default_fields,
)
if policy.ctx.output.use_s3():
records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
records = fs_record_set(policy.ctx.output_path, policy.name)
rows = formatter.to_csv(records)
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def run(self, *args, **kw):
if self.policy.region and (
self.policy.region != self.policy.options.region):
self.policy.log.info(
"Skipping policy %s target-region: %s current-region: %s",
self.policy.name, self.policy.region,
self.policy.options.region)
return
with self.policy.ctx:
self.policy.log.info(
"Running policy %s resource: %s region:%s c7n:%s",
self.policy.name, self.policy.resource_type,
self.policy.options.region,
version)
s = time.time()
resources = self.policy.resource_manager.resources()
rt = time.time() - s
self.policy.log.info(
"policy: %s resource:%s has count:%d time:%0.2f" % (
self.policy.name,
self.policy.resource_type,
len(resources), rt))
self.policy.ctx.metrics.put_metric(
"ResourceCount", len(resources), "Count", Scope="Policy")
self.policy.ctx.metrics.put_metric(
"ResourceTime", rt, "Seconds", Scope="Policy")
self.policy._write_file(
'resources.json', utils.dumps(resources, indent=2))
if not resources:
return []
elif (self.policy.max_resources is not None and
len(resources) > self.policy.max_resources):
msg = "policy %s matched %d resources max resources %s" % (
self.policy.name, len(resources), self.policy.max_resources)
self.policy.log.warning(msg)
raise RuntimeError(msg)
if self.policy.options.dryrun:
self.policy.log.debug("dryrun: skipping actions")
return resources
at = time.time()
for a in self.policy.resource_manager.actions:
s = time.time()
results = a.process(resources)
self.policy.log.info(
"policy: %s action: %s"
" resources: %d"
" execution_time: %0.2f" % (
self.policy.name, a.name,
len(resources), time.time()-s))
self.policy._write_file(
"action-%s" % a.name, utils.dumps(results))
self.policy.ctx.metrics.put_metric(
"ActionTime", time.time() - at, "Seconds", Scope="Policy")
return resources
def run(self, *args, **kw):
if not self.is_runnable():
return
with self.policy.ctx:
self.policy.log.debug(
"Running policy %s resource: %s region:%s c7n:%s",
self.policy.name, self.policy.resource_type,
self.policy.options.region or 'default',
version)
s = time.time()
try:
resources = self.policy.resource_manager.resources()
except ResourceLimitExceeded as e:
self.policy.log.error(str(e))
self.policy.ctx.metrics.put_metric(
'ResourceLimitExceeded', e.selection_count, "Count")
raise
rt = time.time() - s
self.policy.log.info(
"policy: %s resource:%s region:%s count:%d time:%0.2f" % (
self.policy.name,
self.policy.resource_type,
self.policy.options.region,
len(resources), rt))
self.policy.ctx.metrics.put_metric(
"ResourceCount", len(resources), "Count", Scope="Policy")
self.policy.ctx.metrics.put_metric(
"ResourceTime", rt, "Seconds", Scope="Policy")
self.policy._write_file(
'resources.json', utils.dumps(resources, indent=2))
if not resources:
return []
if self.policy.options.dryrun:
self.policy.log.debug("dryrun: skipping actions")
return resources
at = time.time()
for a in self.policy.resource_manager.actions:
s = time.time()
with self.policy.ctx.tracer.subsegment('action:%s' % a.type):
results = a.process(resources)
self.policy.log.info(
"policy: %s action: %s"
" resources: %d"
" execution_time: %0.2f" % (
self.policy.name, a.name,
len(resources), time.time() - s))
if results:
self.policy._write_file(
"action-%s" % a.name, utils.dumps(results))
self.policy.ctx.metrics.put_metric(
"ActionTime", time.time() - at, "Seconds", Scope="Policy")
return resources
def run(self, event, lambda_context):
"""Run policy in push mode against given event.
Lambda automatically generates cloud watch logs, and metrics
for us, albeit with some deficienies, metrics no longer count
against valid resources matches, but against execution.
If metrics execution option is enabled, custodian will generate
metrics per normal.
"""
from c7n.actions import EventAction
mode = self.policy.data.get('mode', {})
if not bool(mode.get("log", True)):
root = logging.getLogger()
map(root.removeHandler, root.handlers[:])
root.handlers = [logging.NullHandler()]
resources = self.resolve_resources(event)
if not resources:
return resources
resources = self.policy.resource_manager.filter_resources(
resources, event)
if 'debug' in event:
self.policy.log.info("Filtered resources %d" % len(resources))
if not resources:
self.policy.log.info(
"policy: %s resources: %s no resources matched" % (
self.policy.name, self.policy.resource_type))
return
with self.policy.ctx:
self.policy.ctx.metrics.put_metric(
'ResourceCount', len(resources), 'Count', Scope="Policy",
buffer=False)
if 'debug' in event:
self.policy.log.info(
"Invoking actions %s", self.policy.resource_manager.actions)
self.policy._write_file(
'resources.json', utils.dumps(resources, indent=2))
for action in self.policy.resource_manager.actions:
self.policy.log.info(
"policy: %s invoking action: %s resources: %d",
self.policy.name, action.name, len(resources))
if isinstance(action, EventAction):
results = action.process(resources, event)
else:
results = action.process(resources)
self.policy._write_file(
"action-%s" % action.name, utils.dumps(results))
return resources
def process(self, resources, event=None):
resources = super(Time, self).process(resources)
if self.parse_errors and self.manager and self.manager.log_dir:
self.log.warning("parse errors %d", len(self.parse_errors))
with open(join(
self.manager.log_dir, 'parse_errors.json'), 'w') as fh:
dumps(self.parse_errors, fh=fh)
self.parse_errors = []
if self.opted_out and self.manager and self.manager.log_dir:
self.log.debug("disabled count %d", len(self.opted_out))
with open(join(
self.manager.log_dir, 'opted_out.json'), 'w') as fh:
dumps(self.opted_out, fh=fh)
self.opted_out = []
return resources
def test_policy_metrics(self):
session_factory = self.replay_flight_data("test_policy_metrics")
p = self.load_policy(
{
"name": "s3-encrypt-keys",
"resource": "s3",
"actions": [{"type": "encrypt-keys"}],
},
session_factory=session_factory,
)
end = datetime.now().replace(hour=0, minute=0, microsecond=0)
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.maxDiff = None
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)),
{
"ActionTime": [
{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 8541.752702140668,
"Sum": 128126.29053211001,
"Unit": "Seconds",
}
],
"Total Keys": [
{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 1575708.7333333334,
"Sum": 23635631.0,
"Unit": "Count",
}
],
"ResourceTime": [
{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 8.682969363532667,
"Sum": 130.24454045299,
"Unit": "Seconds",
}
],
"ResourceCount": [
{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 23.6,
"Sum": 354.0,
"Unit": "Count",
}
],
"Unencrypted": [
{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 10942.266666666666,
"Sum": 164134.0,
"Unit": "Count",
}
],
},
)
def format_resource(self, r):
details = {}
for k in r:
if isinstance(k, (list, dict)):
continue
details[k] = r[k]
for f in self.fields:
value = jmespath.search(f['expr'], r)
if not value:
continue
details[f['key']] = value
for k, v in details.items():
if isinstance(v, datetime):
v = v.isoformat()
elif isinstance(v, (list, dict)):
v = dumps(v)
elif isinstance(v, (int, float, bool)):
v = str(v)
else:
continue
details[k] = v
details['c7n:resource-type'] = self.manager.type
other = {
'Type': 'Other',
'Id': self.manager.get_arns([r])[0],
'Region': self.manager.config.region,
'Details': {'Other': filter_empty(details)}
}
tags = {t['Key']: t['Value'] for t in r.get('Tags', [])}
if tags:
other['Tags'] = tags
return other
def process(self, instances):
instances = self._filter_ec2_with_volumes(
self.filter_instance_state(instances))
if not len(instances):
return
client = utils.local_session(self.manager.session_factory).client('ec2')
failures = {}
# Play nice around aws having insufficient capacity...
for itype, t_instances in utils.group_by(
instances, 'InstanceType').items():
for izone, z_instances in utils.group_by(
t_instances, 'Placement.AvailabilityZone').items():
for batch in utils.chunks(z_instances, self.batch_size):
fails = self.process_instance_set(client, batch, itype, izone)
if fails:
failures["%s %s" % (itype, izone)] = [i['InstanceId'] for i in batch]
if failures:
fail_count = sum(map(len, failures.values()))
msg = "Could not start %d of %d instances %s" % (
fail_count, len(instances),
utils.dumps(failures))
self.log.warning(msg)
raise RuntimeError(msg)
def send_sqs(self, message):
queue = self.data['transport']['queue']
if queue.startswith('https://sqs.'):
region = queue.split('.', 2)[1]
queue_url = queue
elif queue.startswith('arn:sqs'):
queue_arn_split = queue.split(':', 5)
region = queue_arn_split[3]
owner_id = queue_arn_split[4]
queue_name = queue_arn_split[5]
queue_url = "https://sqs.%s.amazonaws.com/%s/%s" % (
region, owner_id, queue_name)
else:
region = self.manager.config.region
owner_id = self.manager.config.account_id
queue_name = queue
queue_url = "https://sqs.%s.amazonaws.com/%s/%s" % (
region, owner_id, queue_name)
client = self.manager.session_factory(
region=region, assume=self.assume_role).client('sqs')
attrs = {
'mtype': {
'DataType': 'String',
'StringValue': self.C7N_DATA_MESSAGE,
},
}
result = client.send_message(
QueueUrl=queue_url,
MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))),
MessageAttributes=attrs)
return result['MessageId']
def test_lambda_policy_metrics(self):
session_factory = self.replay_flight_data('test_lambda_policy_metrics')
p = self.load_policy({
'name': 'ec2-tag-compliance-v6',
'resource': 'ec2',
'mode': {
'type': 'ec2-instance-state',
'events': ['running']},
'filters': [
{"tag:custodian_status": 'absent'},
{'or': [
{"tag:App": 'absent'},
{"tag:Env": 'absent'},
{"tag:Owner": 'absent'}]}]},
session_factory=session_factory)
end = datetime.utcnow()
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)),
{u'Durations': [],
u'Errors': [{u'Sum': 0.0,
u'Timestamp': u'2016-05-30T10:50:00+00:00',
u'Unit': u'Count'}],
u'Invocations': [{u'Sum': 4.0,
u'Timestamp': u'2016-05-30T10:50:00+00:00',
u'Unit': u'Count'}],
u'ResourceCount': [{u'Average': 1.0,
u'Sum': 2.0,
u'Timestamp': u'2016-05-30T10:50:00+00:00',
u'Unit': u'Count'}],
u'Throttles': [{u'Sum': 0.0,
u'Timestamp': u'2016-05-30T10:50:00+00:00',
u'Unit': u'Count'}]})
def process(self, resources, event=None):
client = utils.local_session(
self.manager.session_factory).client('lambda')
params = dict(FunctionName=self.data['function'])
if self.data.get('qualifier'):
params['Qualifier'] = self.data['Qualifier']
if self.data.get('async', True):
params['InvocationType'] = 'Event'
payload = {
'version': VERSION,
'event': event,
'action': self.data,
'policy': self.manager.data}
results = []
for resource_set in utils.chunks(resources, self.data.get('batch_size', 250)):
payload['resources'] = resource_set
params['Payload'] = utils.dumps(payload)
result = client.invoke(**params)
result['Payload'] = result['Payload'].read()
results.append(result)
return results
def metrics_cmd(options, policies):
start, end = _metrics_get_endpoints(options)
data = {}
for p in policies:
log.info('Getting %s metrics', p)
data[p.name] = p.get_metrics(start, end, options.period)
print(dumps(data, indent=2))
def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(
policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1
)
records = []
for policy in policies:
# initialize policy execution context for output access
policy.ctx.initialize()
if policy.ctx.output.type == 's3':
policy_records = record_set(
policy.session_factory,
policy.ctx.output.config['netloc'],
policy.ctx.output.config['path'].strip('/'),
start_date)
else:
policy_records = fs_record_set(policy.ctx.log_dir, policy.name)
log.debug("Found %d records for region %s", len(policy_records), policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = UnicodeWriter(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
elif options.format == 'json':
print(dumps(records, indent=2))
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def run(organization, hook_context, github_url, github_token,
verbose, metrics=False, since=None, assume=None, region=None):
"""scan org repo status hooks"""
logging.basicConfig(level=logging.DEBUG)
since = dateparser.parse(
since, settings={
'RETURN_AS_TIMEZONE_AWARE': True, 'TO_TIMEZONE': 'UTC'})
headers = {"Authorization": "token {}".format(github_token)}
response = requests.post(
github_url, headers=headers,
json={'query': query, 'variables': {'organization': organization}})
result = response.json()
if response.status_code != 200 or 'errors' in result:
raise Exception(
"Query failed to run by returning code of {}. {}".format(
response.status_code, response.content))
now = datetime.utcnow().replace(tzinfo=tzutc())
stats = Counter()
repo_metrics = RepoMetrics(
Bag(session_factory=SessionFactory(region, assume_role=assume)),
{'namespace': DEFAULT_NAMESPACE}
)
for r in result['data']['organization']['repositories']['nodes']:
commits = jmespath.search(
'pullRequests.edges[].node[].commits[].nodes[].commit[]', r)
if not commits:
continue
log.debug("processing repo: %s prs: %d", r['name'], len(commits))
repo_metrics.dims = {
'Hook': hook_context,
'Repo': '{}/{}'.format(organization, r['name'])}
# Each commit represents a separate pr
for c in commits:
process_commit(c, r, repo_metrics, stats, since, now)
repo_metrics.dims = None
if stats['missing']:
repo_metrics.put_metric(
'RepoHookPending', stats['missing'], 'Count',
Hook=hook_context)
repo_metrics.put_metric(
'RepoHookLatency', stats['missing_time'], 'Seconds',
Hook=hook_context)
if not metrics:
print(dumps(repo_metrics.buf, indent=2))
return
else:
repo_metrics.BUF_SIZE = 20
repo_metrics.flush()
def index(self, points):
# account, region in templ
key = self.config['indexer']['template'].format(points[0])
# day aggregation
self.client.put_object(
Bucket=self.config['indexer']['Bucket'],
Key=key,
Body=dumps(points))
def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None):
if isinstance(self.ctx.session_factory, credentials.SessionFactory):
self.ctx.session_factory.set_subscribers(())
self.ctx.metrics.put_metric(
"ApiCalls", sum(self.api_calls.values()), "Count")
self.ctx.policy._write_file(
'api-stats.json', utils.dumps(dict(self.api_calls)))
self.pop_snapshot()
def send_sns(self, message):
topic = self.data['transport']['topic']
region = topic.split(':', 5)[3]
client = self.manager.session_factory(region=region).client('sns')
client.publish(
TopicArn=topic,
Message=base64.b64encode(zlib.compress(utils.dumps(message)))
)
def index(self, metrics_set):
for r, rtype, m, point_set in metrics_set:
mdir = os.path.join(
self.dir, r['account_id'], rtype.id(r))
if not os.path.exists(mdir):
os.makedirs(mdir)
with open(os.path.join(mdir, '%s.json'), 'w') as fh:
fh.write(dumps([r, rtype, m, point_set]))
def test_lambda_policy_metrics(self):
session_factory = self.replay_flight_data("test_lambda_policy_metrics")
p = self.load_policy(
{
"name": "ec2-tag-compliance-v6",
"resource": "ec2",
"mode": {"type": "ec2-instance-state", "events": ["running"]},
"filters": [
{"tag:custodian_status": "absent"},
{
"or": [
{"tag:App": "absent"},
{"tag:Env": "absent"},
{"tag:Owner": "absent"},
]
},
],
},
session_factory=session_factory,
)
end = datetime.utcnow()
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)),
{
u"Durations": [],
u"Errors": [
{
u"Sum": 0.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}
],
u"Invocations": [
{
u"Sum": 4.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}
],
u"ResourceCount": [
{
u"Average": 1.0,
u"Sum": 2.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}
],
u"Throttles": [
{
u"Sum": 0.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}
],
},
)
def metrics_cmd(options, policies):
log.warning("metrics command is deprecated, and will be removed in future")
policies = [p for p in policies if p.provider_name == 'aws']
start, end = _metrics_get_endpoints(options)
data = {}
for p in policies:
log.info('Getting %s metrics', p)
data[p.name] = p.get_metrics(start, end, options.period)
print(dumps(data, indent=2))
def test_lambda_policy_metrics(self):
session_factory = self.replay_flight_data('test_lambda_policy_metrics')
p = self.load_policy(
{
'name':
'ec2-tag-compliance-v6',
'resource':
'ec2',
'mode': {
'type': 'ec2-instance-state',
'events': ['running']
},
'role':
"arn:aws:iam::619193117841:role/CustodianDemoRole",
'filters': [{
"tag:custodian_status": 'absent'
}, {
'or': [{
"tag:App": 'absent'
}, {
"tag:Env": 'absent'
}, {
"tag:Owner": 'absent'
}]
}]
},
session_factory=session_factory)
end = datetime.utcnow()
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)), {
u'Durations': [],
u'Errors': [{
u'Sum': 0.0,
u'Timestamp': u'2016-05-30T10:50:00',
u'Unit': u'Count'
}],
u'Invocations': [{
u'Sum': 4.0,
u'Timestamp': u'2016-05-30T10:50:00',
u'Unit': u'Count'
}],
u'ResourceCount': [{
u'Average': 1.0,
u'Sum': 2.0,
u'Timestamp': u'2016-05-30T10:50:00',
u'Unit': u'Count'
}],
u'Throttles': [{
u'Sum': 0.0,
u'Timestamp': u'2016-05-30T10:50:00',
u'Unit': u'Count'
}]
})
def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(
policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1
)
records = []
for policy in policies:
if policy.ctx.output.use_s3():
policy_records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
policy_records = fs_record_set(policy.ctx.output_path, policy.name)
log.debug("Found %d records for region %s", len(policy_records), policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = UnicodeWriter(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None):
"""Format a policy's extant records into a report."""
regions = set([p.options.region for p in policies])
policy_names = set([p.name for p in policies])
formatter = Formatter(
policies[0].resource_manager.resource_type,
extra_fields=options.field,
include_default_fields=not options.no_default_fields,
include_region=len(regions) > 1,
include_policy=len(policy_names) > 1
)
records = []
for policy in policies:
if policy.ctx.output.use_s3():
policy_records = record_set(
policy.session_factory,
policy.ctx.output.bucket,
policy.ctx.output.key_prefix,
start_date)
else:
policy_records = fs_record_set(policy.ctx.output_path, policy.name)
log.debug("Found %d records for region %s", len(policy_records), policy.options.region)
for record in policy_records:
record['policy'] = policy.name
record['region'] = policy.options.region
records += policy_records
rows = formatter.to_csv(records)
if options.format == 'csv':
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
else:
# We special case CSV, and for other formats we pass to tabulate
print(tabulate(rows, formatter.headers(), tablefmt=options.format))
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def scaleout(*args, **kw):
client = boto3.client('lambda')
client.invoke(
FunctionName=os.environ['AWS_LAMBDA_FUNCTION_NAME'],
InvocationType='Event',
Payload=dumps({
'event': 'fanout',
'function': func.__name__,
'args': args,
'kwargs': kw}),
Qualifier=os.environ['AWS_LAMBDA_FUNCTION_VERSION'])
def scaleout(*args, **kw):
client = boto3.client('lambda')
client.invoke(FunctionName=os.environ['AWS_LAMBDA_FUNCTION_NAME'],
InvocationType='Event',
Payload=dumps({
'event': 'fanout',
'function': func.__name__,
'args': args,
'kwargs': kw
}),
Qualifier=os.environ['AWS_LAMBDA_FUNCTION_VERSION'])
def run_for_event(policy, event=None):
s = time.time()
resources = policy.resource_manager.get_resources(
[AzureModeCommon.extract_resource_id(policy, event)])
resources = policy.resource_manager.filter_resources(
resources, event)
with policy.ctx:
rt = time.time() - s
policy.ctx.metrics.put_metric(
'ResourceCount', len(resources), 'Count', Scope="Policy",
buffer=False)
policy.ctx.metrics.put_metric(
"ResourceTime", rt, "Seconds", Scope="Policy")
policy._write_file(
'resources.json', utils.dumps(resources, indent=2))
if not resources:
policy.log.info(
"policy: %s resources: %s no resources found" % (
policy.name, policy.resource_type))
return
at = time.time()
for action in policy.resource_manager.actions:
policy.log.info(
"policy: %s invoking action: %s resources: %d",
policy.name, action.name, len(resources))
if isinstance(action, EventAction):
results = action.process(resources, event)
else:
results = action.process(resources)
policy._write_file(
"action-%s" % action.name, utils.dumps(results))
policy.ctx.metrics.put_metric(
"ActionTime", time.time() - at, "Seconds", Scope="Policy")
return resources
def poll(self):
"""Query resources and apply policy."""
with self.ctx:
self.log.info("Running policy %s resource: %s region:%s",
self.name, self.resource_type, self.options.region)
s = time.time()
resources = self.resource_manager.resources()
rt = time.time() - s
self.log.info("policy: %s resource:%s has count:%d time:%0.2f" %
(self.name, self.resource_type, len(resources), rt))
self.ctx.metrics.put_metric("ResourceCount",
len(resources),
"Count",
Scope="Policy")
self.ctx.metrics.put_metric("ResourceTime",
rt,
"Seconds",
Scope="Policy")
self._write_file('resources.json', utils.dumps(resources,
indent=2))
if not resources:
return []
if self.options.dryrun:
self.log.debug("dryrun: skipping actions")
return resources
at = time.time()
for a in self.resource_manager.actions:
s = time.time()
results = a.process(resources)
self.log.info(
"policy: %s action: %s resources: %d execution_time: %0.2f"
% (self.name, a.name, len(resources), time.time() - s))
self._write_file("action-%s" % a.name, utils.dumps(results))
self.ctx.metrics.put_metric("ActionTime",
time.time() - at,
"Seconds",
Scope="Policy")
return resources
def run(self, event=None, lambda_context=None):
"""Run the actual policy."""
subscribed_events = AzureEvents.get_event_operations(
self.policy.data['mode'].get('events'))
resource_ids = list(
set([
e['subject'] for e in event
if self._is_subscribed_to_event(e, subscribed_events)
]))
resources = self.policy.resource_manager.get_resources(resource_ids)
if not resources:
self.policy.log.info(
"policy: %s resources: %s no resources found" %
(self.policy.name, self.policy.resource_type))
return
with self.policy.ctx:
self.policy.ctx.metrics.put_metric('ResourceCount',
len(resources),
'Count',
Scope="Policy",
buffer=False)
self.policy._write_file('resources.json',
utils.dumps(resources, indent=2))
for action in self.policy.resource_manager.actions:
self.policy.log.info(
"policy: %s invoking action: %s resources: %d",
self.policy.name, action.name, len(resources))
if isinstance(action, EventAction):
results = action.process(resources, event)
else:
results = action.process(resources)
self.policy._write_file("action-%s" % action.name,
utils.dumps(results))
return resources
def test_policy_metrics(self):
session_factory = self.replay_flight_data("test_policy_metrics")
p = self.load_policy(
{
"name": "s3-encrypt-keys",
"resource": "s3",
"actions": [{
"type": "encrypt-keys"
}],
},
session_factory=session_factory,
)
end = datetime.now().replace(hour=0, minute=0, microsecond=0)
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.maxDiff = None
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)),
{
"ActionTime": [{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 8541.752702140668,
"Sum": 128126.29053211001,
"Unit": "Seconds",
}],
"Total Keys": [{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 1575708.7333333334,
"Sum": 23635631.0,
"Unit": "Count",
}],
"ResourceTime": [{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 8.682969363532667,
"Sum": 130.24454045299,
"Unit": "Seconds",
}],
"ResourceCount": [{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 23.6,
"Sum": 354.0,
"Unit": "Count",
}],
"Unencrypted": [{
"Timestamp": "2016-05-30T00:00:00+00:00",
"Average": 10942.266666666666,
"Sum": 164134.0,
"Unit": "Count",
}],
},
)
def run(self, event=None, lambda_context=None):
"""Run the actual policy."""
resources = self.policy.resource_manager.get_resources(
[event['subject']])
resources = self.policy.resource_manager.filter_resources(
resources, event)
if not resources:
self.policy.log.info(
"policy: %s resources: %s no resources found" %
(self.policy.name, self.policy.resource_type))
return
resources = self.policy.resource_manager.filter_resources(
resources, event)
with self.policy.ctx:
self.policy.ctx.metrics.put_metric('ResourceCount',
len(resources),
'Count',
Scope="Policy",
buffer=False)
self.policy._write_file('resources.json',
utils.dumps(resources, indent=2))
for action in self.policy.resource_manager.actions:
self.policy.log.info(
"policy: %s invoking action: %s resources: %d",
self.policy.name, action.name, len(resources))
if isinstance(action, EventAction):
results = action.process(resources, event)
else:
results = action.process(resources)
self.policy._write_file("action-%s" % action.name,
utils.dumps(results))
return resources
def send_sqs(self, message):
queue = self.data['transport']['queue']
region = self.data['transport'].get('region', 'us-east-1')
client = self.manager.session_factory(region=region).client('sqs')
attrs = {
'mtype': {
'DataType': 'String',
'StringValue': self.C7N_DATA_MESSAGE,
},
}
client.send_message(
QueueUrl=queue,
MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))),
MessageAttributes=attrs)
def send_sqs(self, message):
queue = self.data['transport']['queue']
region = self.data['transport'].get('region', 'us-east-1')
client = self.manager.session_factory(region=region).client('sqs')
attrs = {
'mtype': {
'DataType': 'String',
'StringValue': self.C7N_DATA_MESSAGE,
},
}
client.send_message(
QueueUrl=queue,
MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))),
MessageAttributes=attrs)
def send_sns(self, message):
topic = self.data['transport']['topic']
if topic.startswith('arn:aws:sns'):
region = region = topic.split(':', 5)[3]
topic_arn = topic
else:
region = message['region']
topic_arn = "arn:aws:sns:%s:%s:%s" % (
message['region'], message['account_id'], topic)
client = self.manager.session_factory(
region=region, assume=self.assume_role).client('sns')
client.publish(
TopicArn=topic_arn,
Message=base64.b64encode(zlib.compress(utils.dumps(message))))
def report(policy, start_date, output_fh, raw_output_fh=None, filters=None):
"""Format a policy's extant records into a report."""
formatter = RECORD_TYPE_FORMATTERS.get(policy.resource_type)
if formatter is None:
raise ValueError(
"No formatter for resource type %s, valid: %s" %
(policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS)))
if policy.ctx.output_path.startswith('s3'):
records = record_set(policy.session_factory, policy.ctx.output.bucket,
policy.ctx.output.key_prefix, start_date)
else:
records = fs_record_set(policy.ctx.output_path, policy.name)
rows = formatter.to_csv(records)
writer = csv.writer(output_fh, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
if raw_output_fh is not None:
dumps(records, raw_output_fh, indent=2)
def run(self, dry_run=False, print_only=False):
msg = {
'Body': base64.b64encode(zlib.compress(utils.dumps(self.data))),
'MessageId': 'replayed-message'
}
self.show_to(msg)
if print_only:
self.do_print()
return
if dry_run:
self.do_dry_run(msg)
return
smp = SqsMessageProcessor(self.config, self.session, None, logger)
smp.process_sqs_messsage(msg)
def send_sqs(self, message):
queue = self.data['transport']['queue']
region = queue.split('.', 2)[1]
client = self.manager.session_factory(region=region).client('sqs')
attrs = {
'mtype': {
'DataType': 'String',
'StringValue': self.C7N_DATA_MESSAGE,
},
}
result = client.send_message(
QueueUrl=queue,
MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))),
MessageAttributes=attrs)
return result['MessageId']
def publish_object_records(bid, objects, reporting):
found = False
for k in objects.keys():
if objects[k]:
found = True
if not found:
return
client = get_session({'name': 'object-records'}).client('s3')
bucket = reporting.get('bucket')
record_prefix = reporting.get('record-prefix')
key = "%s/%s/%s/%s.json" % (reporting.get('prefix').strip('/'),
record_prefix, bid, str(uuid4()))
client.put_object(Bucket=bucket,
Key=key,
Body=dumps(objects),
ACL="bucket-owner-full-control",
ServerSideEncryption="AES256")
def test_sqsexec(self):
session_factory = self.replay_flight_data("test_sqs_exec")
client = session_factory().client("sqs")
map_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=map_queue)
reduce_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=reduce_queue)
with SQSExecutor(session_factory, map_queue, reduce_queue) as w:
w.op_sequence_start = 699723
w.op_sequence = 699723
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(client, map_queue, limit=10)
for m in messages:
d = utils.loads(m["Body"])
self.assertEqual(
m["MessageAttributes"]["op"]["StringValue"],
"tests.test_sqsexec:int_processor",
)
client.send_message(
QueueUrl=reduce_queue,
MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]),
MessageAttributes=m["MessageAttributes"],
)
w.gather()
results = [
json.loads(r.result()["Body"]) for r in list(as_completed(futures))
]
self.assertEqual(list(sorted(results))[-1], [[9], 18])
def process(self, instances):
instances = self._filter_ec2_with_volumes(
self.filter_instance_state(instances))
if not len(instances):
return
client = utils.local_session(
self.manager.session_factory).client('ec2')
failures = {}
for batch in utils.chunks(instances, self.batch_size):
fails = self.process_instance_set(client, batch)
if fails:
failures = [i['InstanceId'] for i in batch]
if failures:
fail_count = sum(map(len, failures.values()))
msg = "Could not reboot %d of %d instances %s" % (
fail_count, len(instances), utils.dumps(failures))
self.log.warning(msg)
raise RuntimeError(msg)
def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None):
if exc_type is not None and self.metrics:
self.metrics.put_metric('PolicyException', 1, "Count")
self.policy._write_file('metadata.json',
dumps(self.get_metadata(), indent=2))
self.api_stats.__exit__(exc_type, exc_value, exc_traceback)
with self.tracer.subsegment('output'):
self.metrics.flush()
self.logs.__exit__(exc_type, exc_value, exc_traceback)
self.output.__exit__(exc_type, exc_value, exc_traceback)
self.tracer.__exit__()
self.session_factory.policy_name = None
# IMPORTANT: multi-account execution (c7n-org and others) need
# to manually reset this. Why: Not doing this means we get
# excessive memory usage from client reconstruction for dynamic-gen
# sdks.
if os.environ.get('C7N_TEST_RUN'):
reset_session_cache()
def filter_resources(self, resources, event=None):
original = len(resources)
if event and event.get('debug', False):
self.log.info("Filtering resources using %d filters",
len(self.filters))
for idx, f in enumerate(self.filters, start=1):
if not resources:
break
rcount = len(resources)
with self.ctx.tracer.subsegment("filter:%s" % f.type):
resources = f.process(resources, event)
if event and event.get('debug', False):
self.log.debug("Filter #%d applied %d->%d filter: %s", idx,
rcount, len(resources),
dumps(f.data, indent=None))
self.log.debug(
"Filtered from %d to %d %s" %
(original, len(resources), self.__class__.__name__.lower()))
return resources
def submit(self, func, *args, **kwargs):
self.op_sequence += 1
self.sqs.send_message(
QueueUrl=self.map_queue,
MessageBody=utils.dumps({'args': args, 'kwargs': kwargs}),
MessageAttributes={
'sequence_id': {
'StringValue': str(self.op_sequence),
'DataType': 'Number'},
'op': {
'StringValue': named(func),
'DataType': 'String',
},
'ser': {
'StringValue': 'json',
'DataType': 'String'}}
)
self.futures[self.op_sequence] = f = SQSFuture(
self.op_sequence)
return f
def format_resource(self, r):
details = {}
for k in r:
if isinstance(k, (list, dict)):
continue
details[k] = r[k]
for f in self.fields:
value = jmespath.search(f['expr'], r)
if not value:
continue
details[f['key']] = value
for k, v in details.items():
if isinstance(v, datetime):
v = v.isoformat()
elif isinstance(v, (list, dict)):
v = dumps(v)
elif isinstance(v, (int, float, bool)):
v = str(v)
else:
continue
details[k] = v[:SECHUB_VALUE_SIZE_LIMIT]
details['c7n:resource-type'] = self.manager.type
other = {
'Type': self.resource_type,
'Id': self.manager.get_arns([r])[0],
'Region': self.manager.config.region,
'Partition': get_partition(self.manager.config.region),
'Details': {
self.resource_type: filter_empty(details)
}
}
tags = {t['Key']: t['Value'] for t in r.get('Tags', [])}
if tags:
other['Tags'] = tags
return other
def process(self, resources):
client = local_session(
self.manager.session_factory).client('stepfunctions')
arn = self.data['state-machine']
if not arn.startswith('arn'):
arn = 'arn:aws:states:{}:{}:stateMachine:{}'.format(
self.manager.config.region, self.manager.config.account_id, arn)
params = {'stateMachineArn': arn}
pinput = {}
if self.data.get('policy', True):
pinput['policy'] = dict(self.manager.data)
resource_set = list(zip(self.manager.get_arns(resources), resources))
if self.data.get('bulk', False) is True:
return self.invoke_batch(client, params, pinput, resource_set)
for arn, r in resource_set:
pinput['resource'] = r
params['input'] = dumps(pinput)
r['c7n:execution-arn'] = self.manager.retry(
client.start_execution, **params).get('executionArn')
def process(self, resources, event=None):
params = dict(FunctionName=self.data['function'])
if self.data.get('qualifier'):
params['Qualifier'] = self.data['Qualifier']
if self.data.get('async', True):
params['InvocationType'] = 'Event'
config = Config(read_timeout=self.data.get('timeout', 90),
region_name=self.data.get('region', None))
client = utils.local_session(self.manager.session_factory).client(
'lambda', config=config)
alias = utils.get_account_alias_from_sts(
utils.local_session(self.manager.session_factory))
payload = {
'version': VERSION,
'event': event,
'account_id': self.manager.config.account_id,
'account': alias,
'region': self.manager.config.region,
'action': self.data,
'policy': self.manager.data
}
results = []
for resource_set in utils.chunks(resources,
self.data.get('batch_size', 250)):
payload['resources'] = resource_set
params['Payload'] = utils.dumps(payload)
result = client.invoke(**params)
result['Payload'] = result['Payload'].read()
if isinstance(result['Payload'], bytes):
result['Payload'] = result['Payload'].decode('utf-8')
results.append(result)
return results
def run(self, *args, **kw):
if self.policy.region and (
self.policy.region != self.policy.options.region):
self.policy.log.info(
"Skipping policy %s target-region: %s current-region: %s",
self.policy.name, self.policy.region,
self.policy.options.region)
return
with self.policy.ctx:
self.policy.log.debug(
"Running policy %s resource: %s region:%s c7n:%s",
self.policy.name, self.policy.resource_type,
self.policy.options.region or 'default',
version)
s = time.time()
resources = self.policy.resource_manager.resources()
rt = time.time() - s
self.policy.log.info(
"policy: %s resource:%s region:%s count:%d time:%0.2f" % (
self.policy.name,
self.policy.resource_type,
self.policy.options.region,
len(resources), rt))
self.policy.ctx.metrics.put_metric(
"ResourceCount", len(resources), "Count", Scope="Policy")
self.policy.ctx.metrics.put_metric(
"ResourceTime", rt, "Seconds", Scope="Policy")
self.policy._write_file(
'resources.json', utils.dumps(resources, indent=2))
if not resources:
return []
elif (self.policy.max_resources is not None and
len(resources) > self.policy.max_resources):
msg = "policy %s matched %d resources max resources %s" % (
self.policy.name, len(resources),
self.policy.max_resources)
self.policy.log.warning(msg)
raise RuntimeError(msg)
if self.policy.options.dryrun:
self.policy.log.debug("dryrun: skipping actions")
return resources
at = time.time()
for a in self.policy.resource_manager.actions:
s = time.time()
results = a.process(resources)
self.policy.log.info(
"policy: %s action: %s"
" resources: %d"
" execution_time: %0.2f" % (
self.policy.name, a.name,
len(resources), time.time() - s))
if results:
self.policy._write_file(
"action-%s" % a.name, utils.dumps(results))
self.policy.ctx.metrics.put_metric(
"ActionTime", time.time() - at, "Seconds", Scope="Policy")
return resources
def add(self, keys):
self.count += len(keys)
if self.fh is None:
return
self.fh.write(dumps(keys))
self.fh.write(",\n")
def test_lambda_policy_metrics(self):
session_factory = self.replay_flight_data("test_lambda_policy_metrics")
p = self.load_policy(
{
"name":
"ec2-tag-compliance-v6",
"resource":
"ec2",
"mode": {
"type": "ec2-instance-state",
"events": ["running"]
},
"filters": [
{
"tag:custodian_status": "absent"
},
{
"or": [
{
"tag:App": "absent"
},
{
"tag:Env": "absent"
},
{
"tag:Owner": "absent"
},
]
},
],
},
session_factory=session_factory,
)
end = datetime.utcnow()
start = end - timedelta(14)
period = 24 * 60 * 60 * 14
self.assertEqual(
json.loads(dumps(p.get_metrics(start, end, period), indent=2)),
{
u"Durations": [],
u"Errors": [{
u"Sum": 0.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}],
u"Invocations": [{
u"Sum": 4.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}],
u"ResourceCount": [{
u"Average": 1.0,
u"Sum": 2.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}],
u"Throttles": [{
u"Sum": 0.0,
u"Timestamp": u"2016-05-30T10:50:00+00:00",
u"Unit": u"Count",
}],
},
)
def report(config, output, use, output_dir, accounts,
field, no_default_fields, tags, region, debug, verbose,
policy, policy_tags, format, resource):
"""report on a cross account policy execution."""
accounts_config, custodian_config, executor = init(
config, use, debug, verbose, accounts, tags, policy,
resource=resource, policy_tags=policy_tags)
resource_types = set()
for p in custodian_config.get('policies'):
resource_types.add(p['resource'])
if len(resource_types) > 1:
raise ValueError("can only report on one resource type at a time")
elif not len(custodian_config['policies']) > 0:
raise ValueError("no matching policies found")
records = []
with executor(max_workers=WORKER_COUNT) as w:
futures = {}
for a in accounts_config.get('accounts', ()):
for r in resolve_regions(region or a.get('regions', ())):
futures[w.submit(
report_account,
a, r,
custodian_config,
output_dir,
debug)] = (a, r)
for f in as_completed(futures):
a, r = futures[f]
if f.exception():
if debug:
raise
log.warning(
"Error running policy in %s @ %s exception: %s",
a['name'], r, f.exception())
records.extend(f.result())
log.debug(
"Found %d records across %d accounts and %d policies",
len(records), len(accounts_config['accounts']),
len(custodian_config['policies']))
if format == 'json':
dumps(records, output, indent=2)
return
prefix_fields = OrderedDict(
(('Account', 'account'), ('Region', 'region'), ('Policy', 'policy')))
config = Config.empty()
factory = resource_registry.get(list(resource_types)[0])
formatter = Formatter(
factory.resource_type,
extra_fields=field,
include_default_fields=not(no_default_fields),
include_region=False,
include_policy=False,
fields=prefix_fields)
rows = formatter.to_csv(records, unique=False)
writer = UnicodeWriter(output, formatter.headers())
writer.writerow(formatter.headers())
writer.writerows(rows)
def pack(self, message):
dumped = utils.dumps(message)
compressed = zlib.compress(dumped.encode('utf8'))
b64encoded = base64.b64encode(compressed)
return b64encoded.decode('ascii')
def format_json(self, resources, fh):
return dumps(resources, fh, indent=2)
def test_schema_serialization(self):
try:
dumps(generate())
except Exception:
self.fail("Failed to serialize schema")
def run(organization,
hook_context,
github_url,
github_token,
verbose,
metrics=False,
since=None,
assume=None,
region=None):
"""scan org repo status hooks"""
logging.basicConfig(level=logging.DEBUG)
since = dateparser.parse(since,
settings={
'RETURN_AS_TIMEZONE_AWARE': True,
'TO_TIMEZONE': 'UTC'
})
headers = {"Authorization": "token {}".format(github_token)}
response = requests.post(github_url,
headers=headers,
json={
'query': query,
'variables': {
'organization': organization
}
})
result = response.json()
if response.status_code != 200 or 'errors' in result:
raise Exception(
"Query failed to run by returning code of {}. {}".format(
response.status_code, response.content))
now = datetime.utcnow().replace(tzinfo=tzutc())
stats = Counter()
repo_metrics = RepoMetrics(
Bag(session_factory=SessionFactory(region, assume_role=assume)),
{'namespace': DEFAULT_NAMESPACE})
for r in result['data']['organization']['repositories']['nodes']:
commits = jmespath.search(
'pullRequests.edges[].node[].commits[].nodes[].commit[]', r)
if not commits:
continue
log.debug("processing repo: %s prs: %d", r['name'], len(commits))
repo_metrics.dims = {
'Hook': hook_context,
'Repo': '{}/{}'.format(organization, r['name'])
}
# Each commit represents a separate pr
for c in commits:
process_commit(c, r, repo_metrics, stats, since, now)
repo_metrics.dims = None
if stats['missing']:
repo_metrics.put_metric('RepoHookPending',
stats['missing'],
'Count',
Hook=hook_context)
repo_metrics.put_metric('RepoHookLatency',
stats['missing_time'],
'Seconds',
Hook=hook_context)
if not metrics:
print(dumps(repo_metrics.buf, indent=2))
return
else:
repo_metrics.BUF_SIZE = 20
repo_metrics.flush()
