def report(config, output, use, output_dir, accounts, field, tags, region, debug, verbose, policy, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") records = [] with executor(max_workers=16) as w: futures = {} for a in accounts_config.get('accounts', ()): account_regions = region or a['regions'] for r in account_regions: futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Bag.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=True, include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = csv.writer(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def main(): logging.basicConfig(level=logging.INFO) logging.getLogger('botocore').setLevel(logging.WARNING) parser = setup_parser() options = parser.parse_args() options.log_group = None options.cache = None factory = SessionFactory( options.region, options.profile, options.assume_role) session = factory() client = session.client('cloudwatch') load_resources() policies = load(options, options.config) if options.start and options.end: start = options.start end = options.end elif options.days: end = datetime.utcnow() start = end - timedelta(options.days) data = {} for p in policies: logging.info('Getting %s metrics', p) data[p.name] = p.get_metrics(start, end, options.period) print dumps(data, indent=2)
def report(policy, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" formatter = Formatter( policy.resource_manager, extra_fields=options.field, no_default_fields=options.no_default_fields, ) if policy.ctx.output.use_s3(): records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: records = fs_record_set(policy.ctx.output_path, policy.name) log.debug("Found %d records", len(records)) rows = formatter.to_csv(records) if options.format == 'csv': writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def poll(self): """Query resources and apply policy.""" with self.ctx: self.log.info("Running policy %s" % self.name) s = time.time() resources = self.resource_manager.resources() rt = time.time() - s self.log.info( "policy: %s resource:%s has count:%d time:%0.2f" % ( self.name, self.resource_type, len(resources), rt)) self.ctx.metrics.put_metric( "ResourceCount", len(resources), "Count", Scope="Policy") self.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") self._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: return [] if self.options.dryrun and not self.resource_manager.supports_dry_run: self.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.resource_manager.actions: s = time.time() results = a.process(resources) self.log.info( "policy: %s action: %s resources: %d execution_time: %0.2f" % ( self.name, a.name, len(resources), time.time()-s)) self._write_file("action-%s" % a.name, utils.dumps(results)) self.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def report(policy, start_date, options, output_fh, raw_output_fh=None, filters=None): """Format a policy's extant records into a report.""" formatter_cls = RECORD_TYPE_FORMATTERS.get(policy.resource_type) formatter = formatter_cls( extra_fields=options.field, no_default_fields=options.no_default_fields) if formatter is None: raise ValueError( "No formatter defined for resource type '%s', valid options: %s" % ( policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS))) if policy.ctx.output.use_s3(): records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: records = fs_record_set(policy.ctx.output_path, policy.name) rows = formatter.to_csv(records) writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(policy, start_date, output_fh, raw_output_fh=None, filters=None): """Format a policy's extant records into a report.""" formatter = RECORD_TYPE_FORMATTERS.get(policy.resource_type) if formatter is None: raise ValueError( "No formatter for resource type %s, valid: %s" % ( policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS))) if policy.ctx.output_path.startswith('s3'): records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: records = fs_record_set(policy.ctx.output_path, policy.name) rows = formatter.to_csv(records) writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(policy, start_date, options, output_fh, raw_output_fh=None, filters=None): """Format a policy's extant records into a report.""" if not policy.resource_manager.report_fields: raise ValueError( "No formatter configured for resource type '%s', valid options: %s" % ( policy.resource_type)) formatter = Formatter( policy.resource_manager, extra_fields=options.field, no_default_fields=options.no_default_fields, ) if policy.ctx.output.use_s3(): records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: records = fs_record_set(policy.ctx.output_path, policy.name) rows = formatter.to_csv(records) writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def run(self, *args, **kw): if self.policy.region and ( self.policy.region != self.policy.options.region): self.policy.log.info( "Skipping policy %s target-region: %s current-region: %s", self.policy.name, self.policy.region, self.policy.options.region) return with self.policy.ctx: self.policy.log.info( "Running policy %s resource: %s region:%s c7n:%s", self.policy.name, self.policy.resource_type, self.policy.options.region, version) s = time.time() resources = self.policy.resource_manager.resources() rt = time.time() - s self.policy.log.info( "policy: %s resource:%s has count:%d time:%0.2f" % ( self.policy.name, self.policy.resource_type, len(resources), rt)) self.policy.ctx.metrics.put_metric( "ResourceCount", len(resources), "Count", Scope="Policy") self.policy.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") self.policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: return [] elif (self.policy.max_resources is not None and len(resources) > self.policy.max_resources): msg = "policy %s matched %d resources max resources %s" % ( self.policy.name, len(resources), self.policy.max_resources) self.policy.log.warning(msg) raise RuntimeError(msg) if self.policy.options.dryrun: self.policy.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.policy.resource_manager.actions: s = time.time() results = a.process(resources) self.policy.log.info( "policy: %s action: %s" " resources: %d" " execution_time: %0.2f" % ( self.policy.name, a.name, len(resources), time.time()-s)) self.policy._write_file( "action-%s" % a.name, utils.dumps(results)) self.policy.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def run(self, *args, **kw): if not self.is_runnable(): return with self.policy.ctx: self.policy.log.debug( "Running policy %s resource: %s region:%s c7n:%s", self.policy.name, self.policy.resource_type, self.policy.options.region or 'default', version) s = time.time() try: resources = self.policy.resource_manager.resources() except ResourceLimitExceeded as e: self.policy.log.error(str(e)) self.policy.ctx.metrics.put_metric( 'ResourceLimitExceeded', e.selection_count, "Count") raise rt = time.time() - s self.policy.log.info( "policy: %s resource:%s region:%s count:%d time:%0.2f" % ( self.policy.name, self.policy.resource_type, self.policy.options.region, len(resources), rt)) self.policy.ctx.metrics.put_metric( "ResourceCount", len(resources), "Count", Scope="Policy") self.policy.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") self.policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: return [] if self.policy.options.dryrun: self.policy.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.policy.resource_manager.actions: s = time.time() with self.policy.ctx.tracer.subsegment('action:%s' % a.type): results = a.process(resources) self.policy.log.info( "policy: %s action: %s" " resources: %d" " execution_time: %0.2f" % ( self.policy.name, a.name, len(resources), time.time() - s)) if results: self.policy._write_file( "action-%s" % a.name, utils.dumps(results)) self.policy.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def run(self, event, lambda_context): """Run policy in push mode against given event. Lambda automatically generates cloud watch logs, and metrics for us, albeit with some deficienies, metrics no longer count against valid resources matches, but against execution. If metrics execution option is enabled, custodian will generate metrics per normal. """ from c7n.actions import EventAction mode = self.policy.data.get('mode', {}) if not bool(mode.get("log", True)): root = logging.getLogger() map(root.removeHandler, root.handlers[:]) root.handlers = [logging.NullHandler()] resources = self.resolve_resources(event) if not resources: return resources resources = self.policy.resource_manager.filter_resources( resources, event) if 'debug' in event: self.policy.log.info("Filtered resources %d" % len(resources)) if not resources: self.policy.log.info( "policy: %s resources: %s no resources matched" % ( self.policy.name, self.policy.resource_type)) return with self.policy.ctx: self.policy.ctx.metrics.put_metric( 'ResourceCount', len(resources), 'Count', Scope="Policy", buffer=False) if 'debug' in event: self.policy.log.info( "Invoking actions %s", self.policy.resource_manager.actions) self.policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) for action in self.policy.resource_manager.actions: self.policy.log.info( "policy: %s invoking action: %s resources: %d", self.policy.name, action.name, len(resources)) if isinstance(action, EventAction): results = action.process(resources, event) else: results = action.process(resources) self.policy._write_file( "action-%s" % action.name, utils.dumps(results)) return resources
def process(self, resources, event=None): resources = super(Time, self).process(resources) if self.parse_errors and self.manager and self.manager.log_dir: self.log.warning("parse errors %d", len(self.parse_errors)) with open(join( self.manager.log_dir, 'parse_errors.json'), 'w') as fh: dumps(self.parse_errors, fh=fh) self.parse_errors = [] if self.opted_out and self.manager and self.manager.log_dir: self.log.debug("disabled count %d", len(self.opted_out)) with open(join( self.manager.log_dir, 'opted_out.json'), 'w') as fh: dumps(self.opted_out, fh=fh) self.opted_out = [] return resources
def test_policy_metrics(self): session_factory = self.replay_flight_data("test_policy_metrics") p = self.load_policy( { "name": "s3-encrypt-keys", "resource": "s3", "actions": [{"type": "encrypt-keys"}], }, session_factory=session_factory, ) end = datetime.now().replace(hour=0, minute=0, microsecond=0) start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.maxDiff = None self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), { "ActionTime": [ { "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 8541.752702140668, "Sum": 128126.29053211001, "Unit": "Seconds", } ], "Total Keys": [ { "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 1575708.7333333334, "Sum": 23635631.0, "Unit": "Count", } ], "ResourceTime": [ { "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 8.682969363532667, "Sum": 130.24454045299, "Unit": "Seconds", } ], "ResourceCount": [ { "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 23.6, "Sum": 354.0, "Unit": "Count", } ], "Unencrypted": [ { "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 10942.266666666666, "Sum": 164134.0, "Unit": "Count", } ], }, )
def format_resource(self, r): details = {} for k in r: if isinstance(k, (list, dict)): continue details[k] = r[k] for f in self.fields: value = jmespath.search(f['expr'], r) if not value: continue details[f['key']] = value for k, v in details.items(): if isinstance(v, datetime): v = v.isoformat() elif isinstance(v, (list, dict)): v = dumps(v) elif isinstance(v, (int, float, bool)): v = str(v) else: continue details[k] = v details['c7n:resource-type'] = self.manager.type other = { 'Type': 'Other', 'Id': self.manager.get_arns([r])[0], 'Region': self.manager.config.region, 'Details': {'Other': filter_empty(details)} } tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags: other['Tags'] = tags return other
def process(self, instances): instances = self._filter_ec2_with_volumes( self.filter_instance_state(instances)) if not len(instances): return client = utils.local_session(self.manager.session_factory).client('ec2') failures = {} # Play nice around aws having insufficient capacity... for itype, t_instances in utils.group_by( instances, 'InstanceType').items(): for izone, z_instances in utils.group_by( t_instances, 'Placement.AvailabilityZone').items(): for batch in utils.chunks(z_instances, self.batch_size): fails = self.process_instance_set(client, batch, itype, izone) if fails: failures["%s %s" % (itype, izone)] = [i['InstanceId'] for i in batch] if failures: fail_count = sum(map(len, failures.values())) msg = "Could not start %d of %d instances %s" % ( fail_count, len(instances), utils.dumps(failures)) self.log.warning(msg) raise RuntimeError(msg)
def send_sqs(self, message): queue = self.data['transport']['queue'] if queue.startswith('https://sqs.'): region = queue.split('.', 2)[1] queue_url = queue elif queue.startswith('arn:sqs'): queue_arn_split = queue.split(':', 5) region = queue_arn_split[3] owner_id = queue_arn_split[4] queue_name = queue_arn_split[5] queue_url = "https://sqs.%s.amazonaws.com/%s/%s" % ( region, owner_id, queue_name) else: region = self.manager.config.region owner_id = self.manager.config.account_id queue_name = queue queue_url = "https://sqs.%s.amazonaws.com/%s/%s" % ( region, owner_id, queue_name) client = self.manager.session_factory( region=region, assume=self.assume_role).client('sqs') attrs = { 'mtype': { 'DataType': 'String', 'StringValue': self.C7N_DATA_MESSAGE, }, } result = client.send_message( QueueUrl=queue_url, MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))), MessageAttributes=attrs) return result['MessageId']
def test_lambda_policy_metrics(self): session_factory = self.replay_flight_data('test_lambda_policy_metrics') p = self.load_policy({ 'name': 'ec2-tag-compliance-v6', 'resource': 'ec2', 'mode': { 'type': 'ec2-instance-state', 'events': ['running']}, 'filters': [ {"tag:custodian_status": 'absent'}, {'or': [ {"tag:App": 'absent'}, {"tag:Env": 'absent'}, {"tag:Owner": 'absent'}]}]}, session_factory=session_factory) end = datetime.utcnow() start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), {u'Durations': [], u'Errors': [{u'Sum': 0.0, u'Timestamp': u'2016-05-30T10:50:00+00:00', u'Unit': u'Count'}], u'Invocations': [{u'Sum': 4.0, u'Timestamp': u'2016-05-30T10:50:00+00:00', u'Unit': u'Count'}], u'ResourceCount': [{u'Average': 1.0, u'Sum': 2.0, u'Timestamp': u'2016-05-30T10:50:00+00:00', u'Unit': u'Count'}], u'Throttles': [{u'Sum': 0.0, u'Timestamp': u'2016-05-30T10:50:00+00:00', u'Unit': u'Count'}]})
def process(self, resources, event=None): client = utils.local_session( self.manager.session_factory).client('lambda') params = dict(FunctionName=self.data['function']) if self.data.get('qualifier'): params['Qualifier'] = self.data['Qualifier'] if self.data.get('async', True): params['InvocationType'] = 'Event' payload = { 'version': VERSION, 'event': event, 'action': self.data, 'policy': self.manager.data} results = [] for resource_set in utils.chunks(resources, self.data.get('batch_size', 250)): payload['resources'] = resource_set params['Payload'] = utils.dumps(payload) result = client.invoke(**params) result['Payload'] = result['Payload'].read() results.append(result) return results
def metrics_cmd(options, policies): start, end = _metrics_get_endpoints(options) data = {} for p in policies: log.info('Getting %s metrics', p) data[p.name] = p.get_metrics(start, end, options.period) print(dumps(data, indent=2))
def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter( policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1 ) records = [] for policy in policies: # initialize policy execution context for output access policy.ctx.initialize() if policy.ctx.output.type == 's3': policy_records = record_set( policy.session_factory, policy.ctx.output.config['netloc'], policy.ctx.output.config['path'].strip('/'), start_date) else: policy_records = fs_record_set(policy.ctx.log_dir, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = UnicodeWriter(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) elif options.format == 'json': print(dumps(records, indent=2)) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def run(organization, hook_context, github_url, github_token, verbose, metrics=False, since=None, assume=None, region=None): """scan org repo status hooks""" logging.basicConfig(level=logging.DEBUG) since = dateparser.parse( since, settings={ 'RETURN_AS_TIMEZONE_AWARE': True, 'TO_TIMEZONE': 'UTC'}) headers = {"Authorization": "token {}".format(github_token)} response = requests.post( github_url, headers=headers, json={'query': query, 'variables': {'organization': organization}}) result = response.json() if response.status_code != 200 or 'errors' in result: raise Exception( "Query failed to run by returning code of {}. {}".format( response.status_code, response.content)) now = datetime.utcnow().replace(tzinfo=tzutc()) stats = Counter() repo_metrics = RepoMetrics( Bag(session_factory=SessionFactory(region, assume_role=assume)), {'namespace': DEFAULT_NAMESPACE} ) for r in result['data']['organization']['repositories']['nodes']: commits = jmespath.search( 'pullRequests.edges[].node[].commits[].nodes[].commit[]', r) if not commits: continue log.debug("processing repo: %s prs: %d", r['name'], len(commits)) repo_metrics.dims = { 'Hook': hook_context, 'Repo': '{}/{}'.format(organization, r['name'])} # Each commit represents a separate pr for c in commits: process_commit(c, r, repo_metrics, stats, since, now) repo_metrics.dims = None if stats['missing']: repo_metrics.put_metric( 'RepoHookPending', stats['missing'], 'Count', Hook=hook_context) repo_metrics.put_metric( 'RepoHookLatency', stats['missing_time'], 'Seconds', Hook=hook_context) if not metrics: print(dumps(repo_metrics.buf, indent=2)) return else: repo_metrics.BUF_SIZE = 20 repo_metrics.flush()
def index(self, points): # account, region in templ key = self.config['indexer']['template'].format(points[0]) # day aggregation self.client.put_object( Bucket=self.config['indexer']['Bucket'], Key=key, Body=dumps(points))
def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None): if isinstance(self.ctx.session_factory, credentials.SessionFactory): self.ctx.session_factory.set_subscribers(()) self.ctx.metrics.put_metric( "ApiCalls", sum(self.api_calls.values()), "Count") self.ctx.policy._write_file( 'api-stats.json', utils.dumps(dict(self.api_calls))) self.pop_snapshot()
def send_sns(self, message): topic = self.data['transport']['topic'] region = topic.split(':', 5)[3] client = self.manager.session_factory(region=region).client('sns') client.publish( TopicArn=topic, Message=base64.b64encode(zlib.compress(utils.dumps(message))) )
def index(self, metrics_set): for r, rtype, m, point_set in metrics_set: mdir = os.path.join( self.dir, r['account_id'], rtype.id(r)) if not os.path.exists(mdir): os.makedirs(mdir) with open(os.path.join(mdir, '%s.json'), 'w') as fh: fh.write(dumps([r, rtype, m, point_set]))
def test_lambda_policy_metrics(self): session_factory = self.replay_flight_data("test_lambda_policy_metrics") p = self.load_policy( { "name": "ec2-tag-compliance-v6", "resource": "ec2", "mode": {"type": "ec2-instance-state", "events": ["running"]}, "filters": [ {"tag:custodian_status": "absent"}, { "or": [ {"tag:App": "absent"}, {"tag:Env": "absent"}, {"tag:Owner": "absent"}, ] }, ], }, session_factory=session_factory, ) end = datetime.utcnow() start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), { u"Durations": [], u"Errors": [ { u"Sum": 0.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", } ], u"Invocations": [ { u"Sum": 4.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", } ], u"ResourceCount": [ { u"Average": 1.0, u"Sum": 2.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", } ], u"Throttles": [ { u"Sum": 0.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", } ], }, )
def metrics_cmd(options, policies): log.warning("metrics command is deprecated, and will be removed in future") policies = [p for p in policies if p.provider_name == 'aws'] start, end = _metrics_get_endpoints(options) data = {} for p in policies: log.info('Getting %s metrics', p) data[p.name] = p.get_metrics(start, end, options.period) print(dumps(data, indent=2))
def test_lambda_policy_metrics(self): session_factory = self.replay_flight_data('test_lambda_policy_metrics') p = self.load_policy( { 'name': 'ec2-tag-compliance-v6', 'resource': 'ec2', 'mode': { 'type': 'ec2-instance-state', 'events': ['running'] }, 'role': "arn:aws:iam::619193117841:role/CustodianDemoRole", 'filters': [{ "tag:custodian_status": 'absent' }, { 'or': [{ "tag:App": 'absent' }, { "tag:Env": 'absent' }, { "tag:Owner": 'absent' }] }] }, session_factory=session_factory) end = datetime.utcnow() start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), { u'Durations': [], u'Errors': [{ u'Sum': 0.0, u'Timestamp': u'2016-05-30T10:50:00', u'Unit': u'Count' }], u'Invocations': [{ u'Sum': 4.0, u'Timestamp': u'2016-05-30T10:50:00', u'Unit': u'Count' }], u'ResourceCount': [{ u'Average': 1.0, u'Sum': 2.0, u'Timestamp': u'2016-05-30T10:50:00', u'Unit': u'Count' }], u'Throttles': [{ u'Sum': 0.0, u'Timestamp': u'2016-05-30T10:50:00', u'Unit': u'Count' }] })
def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter( policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1 ) records = [] for policy in policies: if policy.ctx.output.use_s3(): policy_records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: policy_records = fs_record_set(policy.ctx.output_path, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = UnicodeWriter(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def report(policies, start_date, options, output_fh, raw_output_fh=None): """Format a policy's extant records into a report.""" regions = set([p.options.region for p in policies]) policy_names = set([p.name for p in policies]) formatter = Formatter( policies[0].resource_manager.resource_type, extra_fields=options.field, include_default_fields=not options.no_default_fields, include_region=len(regions) > 1, include_policy=len(policy_names) > 1 ) records = [] for policy in policies: if policy.ctx.output.use_s3(): policy_records = record_set( policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: policy_records = fs_record_set(policy.ctx.output_path, policy.name) log.debug("Found %d records for region %s", len(policy_records), policy.options.region) for record in policy_records: record['policy'] = policy.name record['region'] = policy.options.region records += policy_records rows = formatter.to_csv(records) if options.format == 'csv': writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) else: # We special case CSV, and for other formats we pass to tabulate print(tabulate(rows, formatter.headers(), tablefmt=options.format)) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def scaleout(*args, **kw): client = boto3.client('lambda') client.invoke( FunctionName=os.environ['AWS_LAMBDA_FUNCTION_NAME'], InvocationType='Event', Payload=dumps({ 'event': 'fanout', 'function': func.__name__, 'args': args, 'kwargs': kw}), Qualifier=os.environ['AWS_LAMBDA_FUNCTION_VERSION'])
def scaleout(*args, **kw): client = boto3.client('lambda') client.invoke(FunctionName=os.environ['AWS_LAMBDA_FUNCTION_NAME'], InvocationType='Event', Payload=dumps({ 'event': 'fanout', 'function': func.__name__, 'args': args, 'kwargs': kw }), Qualifier=os.environ['AWS_LAMBDA_FUNCTION_VERSION'])
def run_for_event(policy, event=None): s = time.time() resources = policy.resource_manager.get_resources( [AzureModeCommon.extract_resource_id(policy, event)]) resources = policy.resource_manager.filter_resources( resources, event) with policy.ctx: rt = time.time() - s policy.ctx.metrics.put_metric( 'ResourceCount', len(resources), 'Count', Scope="Policy", buffer=False) policy.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: policy.log.info( "policy: %s resources: %s no resources found" % ( policy.name, policy.resource_type)) return at = time.time() for action in policy.resource_manager.actions: policy.log.info( "policy: %s invoking action: %s resources: %d", policy.name, action.name, len(resources)) if isinstance(action, EventAction): results = action.process(resources, event) else: results = action.process(resources) policy._write_file( "action-%s" % action.name, utils.dumps(results)) policy.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def poll(self): """Query resources and apply policy.""" with self.ctx: self.log.info("Running policy %s resource: %s region:%s", self.name, self.resource_type, self.options.region) s = time.time() resources = self.resource_manager.resources() rt = time.time() - s self.log.info("policy: %s resource:%s has count:%d time:%0.2f" % (self.name, self.resource_type, len(resources), rt)) self.ctx.metrics.put_metric("ResourceCount", len(resources), "Count", Scope="Policy") self.ctx.metrics.put_metric("ResourceTime", rt, "Seconds", Scope="Policy") self._write_file('resources.json', utils.dumps(resources, indent=2)) if not resources: return [] if self.options.dryrun: self.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.resource_manager.actions: s = time.time() results = a.process(resources) self.log.info( "policy: %s action: %s resources: %d execution_time: %0.2f" % (self.name, a.name, len(resources), time.time() - s)) self._write_file("action-%s" % a.name, utils.dumps(results)) self.ctx.metrics.put_metric("ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def run(self, event=None, lambda_context=None): """Run the actual policy.""" subscribed_events = AzureEvents.get_event_operations( self.policy.data['mode'].get('events')) resource_ids = list( set([ e['subject'] for e in event if self._is_subscribed_to_event(e, subscribed_events) ])) resources = self.policy.resource_manager.get_resources(resource_ids) if not resources: self.policy.log.info( "policy: %s resources: %s no resources found" % (self.policy.name, self.policy.resource_type)) return with self.policy.ctx: self.policy.ctx.metrics.put_metric('ResourceCount', len(resources), 'Count', Scope="Policy", buffer=False) self.policy._write_file('resources.json', utils.dumps(resources, indent=2)) for action in self.policy.resource_manager.actions: self.policy.log.info( "policy: %s invoking action: %s resources: %d", self.policy.name, action.name, len(resources)) if isinstance(action, EventAction): results = action.process(resources, event) else: results = action.process(resources) self.policy._write_file("action-%s" % action.name, utils.dumps(results)) return resources
def test_policy_metrics(self): session_factory = self.replay_flight_data("test_policy_metrics") p = self.load_policy( { "name": "s3-encrypt-keys", "resource": "s3", "actions": [{ "type": "encrypt-keys" }], }, session_factory=session_factory, ) end = datetime.now().replace(hour=0, minute=0, microsecond=0) start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.maxDiff = None self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), { "ActionTime": [{ "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 8541.752702140668, "Sum": 128126.29053211001, "Unit": "Seconds", }], "Total Keys": [{ "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 1575708.7333333334, "Sum": 23635631.0, "Unit": "Count", }], "ResourceTime": [{ "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 8.682969363532667, "Sum": 130.24454045299, "Unit": "Seconds", }], "ResourceCount": [{ "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 23.6, "Sum": 354.0, "Unit": "Count", }], "Unencrypted": [{ "Timestamp": "2016-05-30T00:00:00+00:00", "Average": 10942.266666666666, "Sum": 164134.0, "Unit": "Count", }], }, )
def run(self, event=None, lambda_context=None): """Run the actual policy.""" resources = self.policy.resource_manager.get_resources( [event['subject']]) resources = self.policy.resource_manager.filter_resources( resources, event) if not resources: self.policy.log.info( "policy: %s resources: %s no resources found" % (self.policy.name, self.policy.resource_type)) return resources = self.policy.resource_manager.filter_resources( resources, event) with self.policy.ctx: self.policy.ctx.metrics.put_metric('ResourceCount', len(resources), 'Count', Scope="Policy", buffer=False) self.policy._write_file('resources.json', utils.dumps(resources, indent=2)) for action in self.policy.resource_manager.actions: self.policy.log.info( "policy: %s invoking action: %s resources: %d", self.policy.name, action.name, len(resources)) if isinstance(action, EventAction): results = action.process(resources, event) else: results = action.process(resources) self.policy._write_file("action-%s" % action.name, utils.dumps(results)) return resources
def send_sqs(self, message): queue = self.data['transport']['queue'] region = self.data['transport'].get('region', 'us-east-1') client = self.manager.session_factory(region=region).client('sqs') attrs = { 'mtype': { 'DataType': 'String', 'StringValue': self.C7N_DATA_MESSAGE, }, } client.send_message( QueueUrl=queue, MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))), MessageAttributes=attrs)
def send_sns(self, message): topic = self.data['transport']['topic'] if topic.startswith('arn:aws:sns'): region = region = topic.split(':', 5)[3] topic_arn = topic else: region = message['region'] topic_arn = "arn:aws:sns:%s:%s:%s" % ( message['region'], message['account_id'], topic) client = self.manager.session_factory( region=region, assume=self.assume_role).client('sns') client.publish( TopicArn=topic_arn, Message=base64.b64encode(zlib.compress(utils.dumps(message))))
def report(policy, start_date, output_fh, raw_output_fh=None, filters=None): """Format a policy's extant records into a report.""" formatter = RECORD_TYPE_FORMATTERS.get(policy.resource_type) if formatter is None: raise ValueError( "No formatter for resource type %s, valid: %s" % (policy.resource_type, ", ".join(RECORD_TYPE_FORMATTERS))) if policy.ctx.output_path.startswith('s3'): records = record_set(policy.session_factory, policy.ctx.output.bucket, policy.ctx.output.key_prefix, start_date) else: records = fs_record_set(policy.ctx.output_path, policy.name) rows = formatter.to_csv(records) writer = csv.writer(output_fh, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows) if raw_output_fh is not None: dumps(records, raw_output_fh, indent=2)
def run(self, dry_run=False, print_only=False): msg = { 'Body': base64.b64encode(zlib.compress(utils.dumps(self.data))), 'MessageId': 'replayed-message' } self.show_to(msg) if print_only: self.do_print() return if dry_run: self.do_dry_run(msg) return smp = SqsMessageProcessor(self.config, self.session, None, logger) smp.process_sqs_messsage(msg)
def send_sqs(self, message): queue = self.data['transport']['queue'] region = queue.split('.', 2)[1] client = self.manager.session_factory(region=region).client('sqs') attrs = { 'mtype': { 'DataType': 'String', 'StringValue': self.C7N_DATA_MESSAGE, }, } result = client.send_message( QueueUrl=queue, MessageBody=base64.b64encode(zlib.compress(utils.dumps(message))), MessageAttributes=attrs) return result['MessageId']
def publish_object_records(bid, objects, reporting): found = False for k in objects.keys(): if objects[k]: found = True if not found: return client = get_session({'name': 'object-records'}).client('s3') bucket = reporting.get('bucket') record_prefix = reporting.get('record-prefix') key = "%s/%s/%s/%s.json" % (reporting.get('prefix').strip('/'), record_prefix, bid, str(uuid4())) client.put_object(Bucket=bucket, Key=key, Body=dumps(objects), ACL="bucket-owner-full-control", ServerSideEncryption="AES256")
def test_sqsexec(self): session_factory = self.replay_flight_data("test_sqs_exec") client = session_factory().client("sqs") map_queue = client.create_queue( QueueName="%s-map-%s" % (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3))) )[ "QueueUrl" ] self.addCleanup(client.delete_queue, QueueUrl=map_queue) reduce_queue = client.create_queue( QueueName="%s-map-%s" % (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3))) )[ "QueueUrl" ] self.addCleanup(client.delete_queue, QueueUrl=reduce_queue) with SQSExecutor(session_factory, map_queue, reduce_queue) as w: w.op_sequence_start = 699723 w.op_sequence = 699723 # Submit work futures = [] for i in range(10): futures.append(w.submit(int_processor, i)) # Manually process and send results messages = MessageIterator(client, map_queue, limit=10) for m in messages: d = utils.loads(m["Body"]) self.assertEqual( m["MessageAttributes"]["op"]["StringValue"], "tests.test_sqsexec:int_processor", ) client.send_message( QueueUrl=reduce_queue, MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]), MessageAttributes=m["MessageAttributes"], ) w.gather() results = [ json.loads(r.result()["Body"]) for r in list(as_completed(futures)) ] self.assertEqual(list(sorted(results))[-1], [[9], 18])
def process(self, instances): instances = self._filter_ec2_with_volumes( self.filter_instance_state(instances)) if not len(instances): return client = utils.local_session( self.manager.session_factory).client('ec2') failures = {} for batch in utils.chunks(instances, self.batch_size): fails = self.process_instance_set(client, batch) if fails: failures = [i['InstanceId'] for i in batch] if failures: fail_count = sum(map(len, failures.values())) msg = "Could not reboot %d of %d instances %s" % ( fail_count, len(instances), utils.dumps(failures)) self.log.warning(msg) raise RuntimeError(msg)
def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None): if exc_type is not None and self.metrics: self.metrics.put_metric('PolicyException', 1, "Count") self.policy._write_file('metadata.json', dumps(self.get_metadata(), indent=2)) self.api_stats.__exit__(exc_type, exc_value, exc_traceback) with self.tracer.subsegment('output'): self.metrics.flush() self.logs.__exit__(exc_type, exc_value, exc_traceback) self.output.__exit__(exc_type, exc_value, exc_traceback) self.tracer.__exit__() self.session_factory.policy_name = None # IMPORTANT: multi-account execution (c7n-org and others) need # to manually reset this. Why: Not doing this means we get # excessive memory usage from client reconstruction for dynamic-gen # sdks. if os.environ.get('C7N_TEST_RUN'): reset_session_cache()
def filter_resources(self, resources, event=None): original = len(resources) if event and event.get('debug', False): self.log.info("Filtering resources using %d filters", len(self.filters)) for idx, f in enumerate(self.filters, start=1): if not resources: break rcount = len(resources) with self.ctx.tracer.subsegment("filter:%s" % f.type): resources = f.process(resources, event) if event and event.get('debug', False): self.log.debug("Filter #%d applied %d->%d filter: %s", idx, rcount, len(resources), dumps(f.data, indent=None)) self.log.debug( "Filtered from %d to %d %s" % (original, len(resources), self.__class__.__name__.lower())) return resources
def submit(self, func, *args, **kwargs): self.op_sequence += 1 self.sqs.send_message( QueueUrl=self.map_queue, MessageBody=utils.dumps({'args': args, 'kwargs': kwargs}), MessageAttributes={ 'sequence_id': { 'StringValue': str(self.op_sequence), 'DataType': 'Number'}, 'op': { 'StringValue': named(func), 'DataType': 'String', }, 'ser': { 'StringValue': 'json', 'DataType': 'String'}} ) self.futures[self.op_sequence] = f = SQSFuture( self.op_sequence) return f
def format_resource(self, r): details = {} for k in r: if isinstance(k, (list, dict)): continue details[k] = r[k] for f in self.fields: value = jmespath.search(f['expr'], r) if not value: continue details[f['key']] = value for k, v in details.items(): if isinstance(v, datetime): v = v.isoformat() elif isinstance(v, (list, dict)): v = dumps(v) elif isinstance(v, (int, float, bool)): v = str(v) else: continue details[k] = v[:SECHUB_VALUE_SIZE_LIMIT] details['c7n:resource-type'] = self.manager.type other = { 'Type': self.resource_type, 'Id': self.manager.get_arns([r])[0], 'Region': self.manager.config.region, 'Partition': get_partition(self.manager.config.region), 'Details': { self.resource_type: filter_empty(details) } } tags = {t['Key']: t['Value'] for t in r.get('Tags', [])} if tags: other['Tags'] = tags return other
def process(self, resources): client = local_session( self.manager.session_factory).client('stepfunctions') arn = self.data['state-machine'] if not arn.startswith('arn'): arn = 'arn:aws:states:{}:{}:stateMachine:{}'.format( self.manager.config.region, self.manager.config.account_id, arn) params = {'stateMachineArn': arn} pinput = {} if self.data.get('policy', True): pinput['policy'] = dict(self.manager.data) resource_set = list(zip(self.manager.get_arns(resources), resources)) if self.data.get('bulk', False) is True: return self.invoke_batch(client, params, pinput, resource_set) for arn, r in resource_set: pinput['resource'] = r params['input'] = dumps(pinput) r['c7n:execution-arn'] = self.manager.retry( client.start_execution, **params).get('executionArn')
def process(self, resources, event=None): params = dict(FunctionName=self.data['function']) if self.data.get('qualifier'): params['Qualifier'] = self.data['Qualifier'] if self.data.get('async', True): params['InvocationType'] = 'Event' config = Config(read_timeout=self.data.get('timeout', 90), region_name=self.data.get('region', None)) client = utils.local_session(self.manager.session_factory).client( 'lambda', config=config) alias = utils.get_account_alias_from_sts( utils.local_session(self.manager.session_factory)) payload = { 'version': VERSION, 'event': event, 'account_id': self.manager.config.account_id, 'account': alias, 'region': self.manager.config.region, 'action': self.data, 'policy': self.manager.data } results = [] for resource_set in utils.chunks(resources, self.data.get('batch_size', 250)): payload['resources'] = resource_set params['Payload'] = utils.dumps(payload) result = client.invoke(**params) result['Payload'] = result['Payload'].read() if isinstance(result['Payload'], bytes): result['Payload'] = result['Payload'].decode('utf-8') results.append(result) return results
def run(self, *args, **kw): if self.policy.region and ( self.policy.region != self.policy.options.region): self.policy.log.info( "Skipping policy %s target-region: %s current-region: %s", self.policy.name, self.policy.region, self.policy.options.region) return with self.policy.ctx: self.policy.log.debug( "Running policy %s resource: %s region:%s c7n:%s", self.policy.name, self.policy.resource_type, self.policy.options.region or 'default', version) s = time.time() resources = self.policy.resource_manager.resources() rt = time.time() - s self.policy.log.info( "policy: %s resource:%s region:%s count:%d time:%0.2f" % ( self.policy.name, self.policy.resource_type, self.policy.options.region, len(resources), rt)) self.policy.ctx.metrics.put_metric( "ResourceCount", len(resources), "Count", Scope="Policy") self.policy.ctx.metrics.put_metric( "ResourceTime", rt, "Seconds", Scope="Policy") self.policy._write_file( 'resources.json', utils.dumps(resources, indent=2)) if not resources: return [] elif (self.policy.max_resources is not None and len(resources) > self.policy.max_resources): msg = "policy %s matched %d resources max resources %s" % ( self.policy.name, len(resources), self.policy.max_resources) self.policy.log.warning(msg) raise RuntimeError(msg) if self.policy.options.dryrun: self.policy.log.debug("dryrun: skipping actions") return resources at = time.time() for a in self.policy.resource_manager.actions: s = time.time() results = a.process(resources) self.policy.log.info( "policy: %s action: %s" " resources: %d" " execution_time: %0.2f" % ( self.policy.name, a.name, len(resources), time.time() - s)) if results: self.policy._write_file( "action-%s" % a.name, utils.dumps(results)) self.policy.ctx.metrics.put_metric( "ActionTime", time.time() - at, "Seconds", Scope="Policy") return resources
def add(self, keys): self.count += len(keys) if self.fh is None: return self.fh.write(dumps(keys)) self.fh.write(",\n")
def test_lambda_policy_metrics(self): session_factory = self.replay_flight_data("test_lambda_policy_metrics") p = self.load_policy( { "name": "ec2-tag-compliance-v6", "resource": "ec2", "mode": { "type": "ec2-instance-state", "events": ["running"] }, "filters": [ { "tag:custodian_status": "absent" }, { "or": [ { "tag:App": "absent" }, { "tag:Env": "absent" }, { "tag:Owner": "absent" }, ] }, ], }, session_factory=session_factory, ) end = datetime.utcnow() start = end - timedelta(14) period = 24 * 60 * 60 * 14 self.assertEqual( json.loads(dumps(p.get_metrics(start, end, period), indent=2)), { u"Durations": [], u"Errors": [{ u"Sum": 0.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", }], u"Invocations": [{ u"Sum": 4.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", }], u"ResourceCount": [{ u"Average": 1.0, u"Sum": 2.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", }], u"Throttles": [{ u"Sum": 0.0, u"Timestamp": u"2016-05-30T10:50:00+00:00", u"Unit": u"Count", }], }, )
def report(config, output, use, output_dir, accounts, field, no_default_fields, tags, region, debug, verbose, policy, policy_tags, format, resource): """report on a cross account policy execution.""" accounts_config, custodian_config, executor = init( config, use, debug, verbose, accounts, tags, policy, resource=resource, policy_tags=policy_tags) resource_types = set() for p in custodian_config.get('policies'): resource_types.add(p['resource']) if len(resource_types) > 1: raise ValueError("can only report on one resource type at a time") elif not len(custodian_config['policies']) > 0: raise ValueError("no matching policies found") records = [] with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config.get('accounts', ()): for r in resolve_regions(region or a.get('regions', ())): futures[w.submit( report_account, a, r, custodian_config, output_dir, debug)] = (a, r) for f in as_completed(futures): a, r = futures[f] if f.exception(): if debug: raise log.warning( "Error running policy in %s @ %s exception: %s", a['name'], r, f.exception()) records.extend(f.result()) log.debug( "Found %d records across %d accounts and %d policies", len(records), len(accounts_config['accounts']), len(custodian_config['policies'])) if format == 'json': dumps(records, output, indent=2) return prefix_fields = OrderedDict( (('Account', 'account'), ('Region', 'region'), ('Policy', 'policy'))) config = Config.empty() factory = resource_registry.get(list(resource_types)[0]) formatter = Formatter( factory.resource_type, extra_fields=field, include_default_fields=not(no_default_fields), include_region=False, include_policy=False, fields=prefix_fields) rows = formatter.to_csv(records, unique=False) writer = UnicodeWriter(output, formatter.headers()) writer.writerow(formatter.headers()) writer.writerows(rows)
def pack(self, message): dumped = utils.dumps(message) compressed = zlib.compress(dumped.encode('utf8')) b64encoded = base64.b64encode(compressed) return b64encoded.decode('ascii')
def format_json(self, resources, fh): return dumps(resources, fh, indent=2)
def test_schema_serialization(self): try: dumps(generate()) except Exception: self.fail("Failed to serialize schema")
def run(organization, hook_context, github_url, github_token, verbose, metrics=False, since=None, assume=None, region=None): """scan org repo status hooks""" logging.basicConfig(level=logging.DEBUG) since = dateparser.parse(since, settings={ 'RETURN_AS_TIMEZONE_AWARE': True, 'TO_TIMEZONE': 'UTC' }) headers = {"Authorization": "token {}".format(github_token)} response = requests.post(github_url, headers=headers, json={ 'query': query, 'variables': { 'organization': organization } }) result = response.json() if response.status_code != 200 or 'errors' in result: raise Exception( "Query failed to run by returning code of {}. {}".format( response.status_code, response.content)) now = datetime.utcnow().replace(tzinfo=tzutc()) stats = Counter() repo_metrics = RepoMetrics( Bag(session_factory=SessionFactory(region, assume_role=assume)), {'namespace': DEFAULT_NAMESPACE}) for r in result['data']['organization']['repositories']['nodes']: commits = jmespath.search( 'pullRequests.edges[].node[].commits[].nodes[].commit[]', r) if not commits: continue log.debug("processing repo: %s prs: %d", r['name'], len(commits)) repo_metrics.dims = { 'Hook': hook_context, 'Repo': '{}/{}'.format(organization, r['name']) } # Each commit represents a separate pr for c in commits: process_commit(c, r, repo_metrics, stats, since, now) repo_metrics.dims = None if stats['missing']: repo_metrics.put_metric('RepoHookPending', stats['missing'], 'Count', Hook=hook_context) repo_metrics.put_metric('RepoHookLatency', stats['missing_time'], 'Seconds', Hook=hook_context) if not metrics: print(dumps(repo_metrics.buf, indent=2)) return else: repo_metrics.BUF_SIZE = 20 repo_metrics.flush()