def __init__(self, data=None, manager=None, log_dir=None): super(AutoTagUser, self).__init__(data, manager, log_dir) delta_days = self.data.get('days', self.max_query_days) self.start_time = utcnow() - datetime.timedelta(days=delta_days) self.client = self.manager.get_client( 'azure.mgmt.monitor.MonitorManagementClient') self.tag_action = self.manager.action_registry.get('tag')
def _query_costs(self): client = self.manager.get_client('azure.mgmt.costmanagement.CostManagementClient') aggregation = {'totalCost': QueryAggregation(name='PreTaxCost')} grouping = [QueryGrouping(type='Dimension', name='ResourceId')] dataset = QueryDataset(grouping=grouping, aggregation=aggregation) timeframe = self.data['timeframe'] time_period = None if timeframe not in CostFilter.preset_timeframes: end_time = utcnow().replace(hour=0, minute=0, second=0, microsecond=0) start_time = end_time - timedelta(days=timeframe) timeframe = 'Custom' time_period = QueryTimePeriod(from_property=start_time, to=end_time) definition = QueryDefinition(timeframe=timeframe, time_period=time_period, dataset=dataset) subscription_id = self.manager.get_session().subscription_id scope = '/subscriptions/' + subscription_id query = client.query.usage_by_scope(scope, definition) if hasattr(query, '_derserializer'): original = query._derserializer._deserialize query._derserializer._deserialize = lambda target, data: \ original(target, self.fix_wrap_rest_response(data)) result = list(query)[0] result = [{result.columns[i].name: v for i, v in enumerate(row)} for row in result.rows] result = {r['ResourceId'].lower(): r for r in result} return result
def _verify_expected_call(self, mock, timeframe, resource_group): subscription_id = self.session.get_subscription_id() mock.assert_called_once() definition = mock.call_args[0][1] if isinstance(timeframe, int): today = utcnow().replace(hour=0, minute=0, second=0, microsecond=0) self.assertEqual(definition.timeframe, 'Custom') self.assertEqual(definition.time_period.to, today) self.assertEqual(definition.time_period.from_property, today - datetime.timedelta(days=timeframe)) else: self.assertEqual(definition.timeframe, timeframe) self.assertEqual(len(definition.dataset.grouping), 1) self.assertEqual(definition.dataset.grouping[0].type, 'Dimension') self.assertEqual( definition.dataset.grouping[0].name, 'ResourceGroupName' if resource_group else 'ResourceId') self.assertEqual(definition.dataset.aggregation['totalCost'].name, 'PreTaxCost') if not resource_group: self.assertEqual(definition.dataset.filter.dimension.name, 'ResourceType') self.assertEqual(definition.dataset.filter.dimension.operator, 'In') self.assertEqual(definition.dataset.filter.dimension.values, ['Microsoft.Compute/virtualMachines']) mock.assert_called_once_with('/subscriptions/' + subscription_id, definition)
def _get_user_from_resource_logs(self, resource): # Makes patching this easier from c7n_azure.utils import utcnow # Calculate start time delta_days = self.data.get('days', self.max_query_days) start_time = utcnow() - datetime.timedelta(days=delta_days) # resource group type if self.manager.type == 'resourcegroup': resource_type = "Microsoft.Resources/subscriptions/resourcegroups" query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceGroupName eq '%s'" % resource['name'], "eventChannels eq 'Operation'" ]) # other Azure resources else: resource_type = resource['type'] query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceUri eq '%s'" % resource['id'], "eventChannels eq 'Operation'" ]) # fetch activity logs logs = self.client.activity_logs.list(filter=query_filter, select=self.query_select) # get the user who issued the first operation operation_name = "%s/write" % resource_type first_op = self.get_first_operation(logs, operation_name) return first_op.caller if first_op else None
def _get_user_from_resource_logs(self, resource): # Calculate start time delta_days = self.data.get('days', self.max_query_days) start_time = utcnow() - datetime.timedelta(days=delta_days) # resource group type if self.manager.type == 'resourcegroup': resource_type = "Microsoft.Resources/subscriptions/resourcegroups" query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceGroupName eq '%s'" % resource['name'], "eventChannels eq 'Operation'" ]) # other Azure resources else: resource_type = resource['type'] query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceUri eq '%s'" % resource['id'], "eventChannels eq 'Operation'" ]) # fetch activity logs logs = self.client.activity_logs.list( filter=query_filter, select=self.query_select ) # get the user who issued the first operation operation_name = "%s/write" % resource_type first_op = self.get_first_operation(logs, operation_name) return first_op.caller if first_op else None
def process_resource(self, resource, event_item=None): # if the auto-tag-user policy set update to False (or it's unset) then we # will skip writing their UserName tag and not overwrite pre-existing values if not self.should_update and resource.get('tags', {}).get(self.tag_key, None): return user = self.default_user if event_item: principal_type = self.principal_type_jmes_path.search(event_item) if principal_type == 'User': user = self.user_jmes_path.search(event_item) or user elif principal_type == 'ServicePrincipal': user = self.sp_jmes_path.search(event_item) or user else: self.log.error('Principal type of event cannot be determined.') return else: # Calculate start time delta_days = self.data.get('days', self.max_query_days) start_time = utcnow() - datetime.timedelta(days=delta_days) # resource group type if self.manager.type == 'resourcegroup': resource_type = "Microsoft.Resources/subscriptions/resourcegroups" query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceGroupName eq '%s'" % resource['name'], "eventChannels eq 'Operation'" ]) # other Azure resources else: resource_type = resource['type'] query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceUri eq '%s'" % resource['id'], "eventChannels eq 'Operation'" ]) # fetch activity logs logs = self.client.activity_logs.list( filter=query_filter, select=self.query_select ) # get the user who issued the first operation operation_name = "%s/write" % resource_type first_op = self.get_first_operation(logs, operation_name) if first_op is not None: user = first_op.caller # issue tag action to label user try: TagHelper.add_tags(self, resource, {self.tag_key: user}) except CloudError as e: # resources can be locked if e.inner_exception.error == 'ScopeLocked': pass
def _query_costs(self): manager = self.manager is_resource_group = manager.type == 'resourcegroup' client = manager.get_client('azure.mgmt.costmanagement.CostManagementClient') aggregation = {'totalCost': QueryAggregation(name='PreTaxCost', function='Sum')} grouping = [QueryGrouping(type='Dimension', name='ResourceGroupName' if is_resource_group else 'ResourceId')] query_filter = None if not is_resource_group: query_filter = QueryFilter( dimension=QueryComparisonExpression(name='ResourceType', operator='In', values=[manager.resource_type.resource_type])) if 'dimension' in query_filter._attribute_map: query_filter._attribute_map['dimension']['key'] = 'dimensions' dataset = QueryDataset(grouping=grouping, aggregation=aggregation, filter=query_filter) timeframe = self.data['timeframe'] time_period = None if timeframe not in CostFilter.preset_timeframes: end_time = utcnow().replace(hour=0, minute=0, second=0, microsecond=0) start_time = end_time - timedelta(days=timeframe) timeframe = 'Custom' time_period = QueryTimePeriod(from_property=start_time, to=end_time) definition = QueryDefinition(type='ActualCost', timeframe=timeframe, time_period=time_period, dataset=dataset) subscription_id = manager.get_session().get_subscription_id() scope = '/subscriptions/' + subscription_id query = client.query.usage(scope, definition) if hasattr(query, '_derserializer'): original = query._derserializer._deserialize query._derserializer._deserialize = lambda target, data: \ original(target, self.fix_wrap_rest_response(data)) result_list = [{query.columns[i].name: v for i, v in enumerate(row)} for row in query.rows] for r in result_list: if 'ResourceGroupName' in r: r['ResourceId'] = scope + '/resourcegroups/' + r.pop('ResourceGroupName') r['ResourceId'] = r['ResourceId'].lower() return result_list
def test_mark_for_op(self): self._run_policy([{ 'type': 'mark-for-op', 'tag': 'cctest_mark', 'op': 'delete', 'msg': '{op}, {action_date}', 'days': self.DAYS }]) expected_date = utils.utcnow() + datetime.timedelta(days=self.DAYS) expected = 'delete, ' + expected_date.strftime('%Y/%m/%d') self.assertEqual(self._get_tags().get('cctest_mark'), expected)
def process_resource(self, resource): # if the auto-tag-user policy set update to False (or it's unset) then we # will skip writing their UserName tag and not overwrite pre-existing values if not self.should_update and resource.get('tags', {}).get(self.tag_key, None): return user = self.default_user # Calculate start time delta_days = self.data.get('days', self.max_query_days) start_time = utcnow() - datetime.timedelta(days=delta_days) # resource group type if self.manager.type == 'resourcegroup': resource_type = "Microsoft.Resources/subscriptions/resourcegroups" query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceGroupName eq '%s'" % resource['name'], "eventChannels eq 'Operation'" ]) # other Azure resources else: resource_type = resource['type'] query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceUri eq '%s'" % resource['id'], "eventChannels eq 'Operation'" ]) # fetch activity logs logs = self.client.activity_logs.list( filter=query_filter, select=self.query_select ) # get the user who issued the first operation operation_name = "%s/write" % resource_type first_op = self.get_first_operation(logs, operation_name) if first_op is not None: user = first_op.caller # issue tag action to label user try: TagHelper.add_tags(self, resource, {self.tag_key: user}) except CloudError as e: # resources can be locked if e.inner_exception.error == 'ScopeLocked': pass
def process(self, resources, event=None): # Import utcnow function as it may have been overridden for testing purposes from c7n_azure.utils import utcnow # Get timespan end_time = utcnow() start_time = end_time - timedelta(hours=self.timeframe) self.timespan = "{}/{}".format(start_time, end_time) # Create Azure Monitor client self.client = self.manager.get_client('azure.mgmt.monitor.MonitorManagementClient') # Process each resource in a separate thread, returning all that pass filter with self.executor_factory(max_workers=3) as w: processed = list(w.map(self.process_resource, resources)) return [item for item in processed if item is not None]
def test_mark_for_op(self, update_resource_tags): action = self._get_action({'op': 'stop', 'days': self.DAYS}) resource = tools.get_resource(self.existing_tags) action.process([resource]) tags = tools.get_tags_parameter(update_resource_tags) date = (utils.utcnow() + datetime.timedelta(days=self.DAYS)).strftime('%Y/%m/%d') expected_value = TagDelayedAction.default_template.format( op='stop', action_date=date) expected_tags = self.existing_tags.copy() expected_tags.update({'custodian_status': expected_value}) self.assertEqual(tags, expected_tags)
def _get_first_event(self, resource): if 'c7n:first_iam_event' in resource: return resource['c7n:first_iam_event'] # Makes patching this easier from c7n_azure.utils import utcnow # Calculate start time delta_days = self.data.get('days', self.max_query_days) start_time = utcnow() - datetime.timedelta(days=delta_days) # resource group type if self.manager.type == 'resourcegroup': resource_type = "Microsoft.Resources/subscriptions/resourcegroups" query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceGroupName eq '%s'" % resource['name'], "eventChannels eq 'Operation'", "resourceType eq '%s'" % resource_type ]) # other Azure resources else: resource_type = resource['type'] query_filter = " and ".join([ "eventTimestamp ge '%s'" % start_time, "resourceUri eq '%s'" % resource['id'], "eventChannels eq 'Operation'", "resourceType eq '%s'" % resource_type ]) # fetch activity logs logs = self.client.activity_logs.list( filter=query_filter, select=self.query_select ) # get the user who issued the first operation operation_name = "%s/write" % resource_type first_event = None for l in logs: if l.operation_name.value and l.operation_name.value.lower() == operation_name.lower(): first_event = l resource['c7n:first_iam_event'] = first_event return first_event