def __init__(self, data=None, manager=None, log_dir=None):
super(AutoTagUser, self).__init__(data, manager, log_dir)
delta_days = self.data.get('days', self.max_query_days)
self.start_time = utcnow() - datetime.timedelta(days=delta_days)
self.client = self.manager.get_client(
'azure.mgmt.monitor.MonitorManagementClient')
self.tag_action = self.manager.action_registry.get('tag')
def _query_costs(self):
client = self.manager.get_client('azure.mgmt.costmanagement.CostManagementClient')
aggregation = {'totalCost': QueryAggregation(name='PreTaxCost')}
grouping = [QueryGrouping(type='Dimension', name='ResourceId')]
dataset = QueryDataset(grouping=grouping, aggregation=aggregation)
timeframe = self.data['timeframe']
time_period = None
if timeframe not in CostFilter.preset_timeframes:
end_time = utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=timeframe)
timeframe = 'Custom'
time_period = QueryTimePeriod(from_property=start_time, to=end_time)
definition = QueryDefinition(timeframe=timeframe, time_period=time_period, dataset=dataset)
subscription_id = self.manager.get_session().subscription_id
scope = '/subscriptions/' + subscription_id
query = client.query.usage_by_scope(scope, definition)
if hasattr(query, '_derserializer'):
original = query._derserializer._deserialize
query._derserializer._deserialize = lambda target, data: \
original(target, self.fix_wrap_rest_response(data))
result = list(query)[0]
result = [{result.columns[i].name: v for i, v in enumerate(row)} for row in result.rows]
result = {r['ResourceId'].lower(): r for r in result}
return result
def _verify_expected_call(self, mock, timeframe, resource_group):
subscription_id = self.session.get_subscription_id()
mock.assert_called_once()
definition = mock.call_args[0][1]
if isinstance(timeframe, int):
today = utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
self.assertEqual(definition.timeframe, 'Custom')
self.assertEqual(definition.time_period.to, today)
self.assertEqual(definition.time_period.from_property,
today - datetime.timedelta(days=timeframe))
else:
self.assertEqual(definition.timeframe, timeframe)
self.assertEqual(len(definition.dataset.grouping), 1)
self.assertEqual(definition.dataset.grouping[0].type, 'Dimension')
self.assertEqual(
definition.dataset.grouping[0].name,
'ResourceGroupName' if resource_group else 'ResourceId')
self.assertEqual(definition.dataset.aggregation['totalCost'].name,
'PreTaxCost')
if not resource_group:
self.assertEqual(definition.dataset.filter.dimension.name,
'ResourceType')
self.assertEqual(definition.dataset.filter.dimension.operator,
'In')
self.assertEqual(definition.dataset.filter.dimension.values,
['Microsoft.Compute/virtualMachines'])
mock.assert_called_once_with('/subscriptions/' + subscription_id,
definition)
def _get_user_from_resource_logs(self, resource):
# Makes patching this easier
from c7n_azure.utils import utcnow
# Calculate start time
delta_days = self.data.get('days', self.max_query_days)
start_time = utcnow() - datetime.timedelta(days=delta_days)
# resource group type
if self.manager.type == 'resourcegroup':
resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceGroupName eq '%s'" % resource['name'],
"eventChannels eq 'Operation'"
])
# other Azure resources
else:
resource_type = resource['type']
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceUri eq '%s'" % resource['id'],
"eventChannels eq 'Operation'"
])
# fetch activity logs
logs = self.client.activity_logs.list(filter=query_filter,
select=self.query_select)
# get the user who issued the first operation
operation_name = "%s/write" % resource_type
first_op = self.get_first_operation(logs, operation_name)
return first_op.caller if first_op else None
def _get_user_from_resource_logs(self, resource):
# Calculate start time
delta_days = self.data.get('days', self.max_query_days)
start_time = utcnow() - datetime.timedelta(days=delta_days)
# resource group type
if self.manager.type == 'resourcegroup':
resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceGroupName eq '%s'" % resource['name'],
"eventChannels eq 'Operation'"
])
# other Azure resources
else:
resource_type = resource['type']
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceUri eq '%s'" % resource['id'],
"eventChannels eq 'Operation'"
])
# fetch activity logs
logs = self.client.activity_logs.list(
filter=query_filter,
select=self.query_select
)
# get the user who issued the first operation
operation_name = "%s/write" % resource_type
first_op = self.get_first_operation(logs, operation_name)
return first_op.caller if first_op else None
def process_resource(self, resource, event_item=None):
# if the auto-tag-user policy set update to False (or it's unset) then we
# will skip writing their UserName tag and not overwrite pre-existing values
if not self.should_update and resource.get('tags', {}).get(self.tag_key, None):
return
user = self.default_user
if event_item:
principal_type = self.principal_type_jmes_path.search(event_item)
if principal_type == 'User':
user = self.user_jmes_path.search(event_item) or user
elif principal_type == 'ServicePrincipal':
user = self.sp_jmes_path.search(event_item) or user
else:
self.log.error('Principal type of event cannot be determined.')
return
else:
# Calculate start time
delta_days = self.data.get('days', self.max_query_days)
start_time = utcnow() - datetime.timedelta(days=delta_days)
# resource group type
if self.manager.type == 'resourcegroup':
resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceGroupName eq '%s'" % resource['name'],
"eventChannels eq 'Operation'"
])
# other Azure resources
else:
resource_type = resource['type']
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceUri eq '%s'" % resource['id'],
"eventChannels eq 'Operation'"
])
# fetch activity logs
logs = self.client.activity_logs.list(
filter=query_filter,
select=self.query_select
)
# get the user who issued the first operation
operation_name = "%s/write" % resource_type
first_op = self.get_first_operation(logs, operation_name)
if first_op is not None:
user = first_op.caller
# issue tag action to label user
try:
TagHelper.add_tags(self, resource, {self.tag_key: user})
except CloudError as e:
# resources can be locked
if e.inner_exception.error == 'ScopeLocked':
pass
def _query_costs(self):
manager = self.manager
is_resource_group = manager.type == 'resourcegroup'
client = manager.get_client('azure.mgmt.costmanagement.CostManagementClient')
aggregation = {'totalCost': QueryAggregation(name='PreTaxCost', function='Sum')}
grouping = [QueryGrouping(type='Dimension',
name='ResourceGroupName' if is_resource_group else 'ResourceId')]
query_filter = None
if not is_resource_group:
query_filter = QueryFilter(
dimension=QueryComparisonExpression(name='ResourceType',
operator='In',
values=[manager.resource_type.resource_type]))
if 'dimension' in query_filter._attribute_map:
query_filter._attribute_map['dimension']['key'] = 'dimensions'
dataset = QueryDataset(grouping=grouping, aggregation=aggregation, filter=query_filter)
timeframe = self.data['timeframe']
time_period = None
if timeframe not in CostFilter.preset_timeframes:
end_time = utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=timeframe)
timeframe = 'Custom'
time_period = QueryTimePeriod(from_property=start_time, to=end_time)
definition = QueryDefinition(type='ActualCost',
timeframe=timeframe,
time_period=time_period,
dataset=dataset)
subscription_id = manager.get_session().get_subscription_id()
scope = '/subscriptions/' + subscription_id
query = client.query.usage(scope, definition)
if hasattr(query, '_derserializer'):
original = query._derserializer._deserialize
query._derserializer._deserialize = lambda target, data: \
original(target, self.fix_wrap_rest_response(data))
result_list = [{query.columns[i].name: v for i, v in enumerate(row)}
for row in query.rows]
for r in result_list:
if 'ResourceGroupName' in r:
r['ResourceId'] = scope + '/resourcegroups/' + r.pop('ResourceGroupName')
r['ResourceId'] = r['ResourceId'].lower()
return result_list
def test_mark_for_op(self):
self._run_policy([{
'type': 'mark-for-op',
'tag': 'cctest_mark',
'op': 'delete',
'msg': '{op}, {action_date}',
'days': self.DAYS
}])
expected_date = utils.utcnow() + datetime.timedelta(days=self.DAYS)
expected = 'delete, ' + expected_date.strftime('%Y/%m/%d')
self.assertEqual(self._get_tags().get('cctest_mark'), expected)
def process_resource(self, resource):
# if the auto-tag-user policy set update to False (or it's unset) then we
# will skip writing their UserName tag and not overwrite pre-existing values
if not self.should_update and resource.get('tags', {}).get(self.tag_key, None):
return
user = self.default_user
# Calculate start time
delta_days = self.data.get('days', self.max_query_days)
start_time = utcnow() - datetime.timedelta(days=delta_days)
# resource group type
if self.manager.type == 'resourcegroup':
resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceGroupName eq '%s'" % resource['name'],
"eventChannels eq 'Operation'"
])
# other Azure resources
else:
resource_type = resource['type']
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceUri eq '%s'" % resource['id'],
"eventChannels eq 'Operation'"
])
# fetch activity logs
logs = self.client.activity_logs.list(
filter=query_filter,
select=self.query_select
)
# get the user who issued the first operation
operation_name = "%s/write" % resource_type
first_op = self.get_first_operation(logs, operation_name)
if first_op is not None:
user = first_op.caller
# issue tag action to label user
try:
TagHelper.add_tags(self, resource, {self.tag_key: user})
except CloudError as e:
# resources can be locked
if e.inner_exception.error == 'ScopeLocked':
pass
def process(self, resources, event=None):
# Import utcnow function as it may have been overridden for testing purposes
from c7n_azure.utils import utcnow
# Get timespan
end_time = utcnow()
start_time = end_time - timedelta(hours=self.timeframe)
self.timespan = "{}/{}".format(start_time, end_time)
# Create Azure Monitor client
self.client = self.manager.get_client('azure.mgmt.monitor.MonitorManagementClient')
# Process each resource in a separate thread, returning all that pass filter
with self.executor_factory(max_workers=3) as w:
processed = list(w.map(self.process_resource, resources))
return [item for item in processed if item is not None]
def test_mark_for_op(self, update_resource_tags):
action = self._get_action({'op': 'stop', 'days': self.DAYS})
resource = tools.get_resource(self.existing_tags)
action.process([resource])
tags = tools.get_tags_parameter(update_resource_tags)
date = (utils.utcnow() +
datetime.timedelta(days=self.DAYS)).strftime('%Y/%m/%d')
expected_value = TagDelayedAction.default_template.format(
op='stop', action_date=date)
expected_tags = self.existing_tags.copy()
expected_tags.update({'custodian_status': expected_value})
self.assertEqual(tags, expected_tags)
def _get_first_event(self, resource):
if 'c7n:first_iam_event' in resource:
return resource['c7n:first_iam_event']
# Makes patching this easier
from c7n_azure.utils import utcnow
# Calculate start time
delta_days = self.data.get('days', self.max_query_days)
start_time = utcnow() - datetime.timedelta(days=delta_days)
# resource group type
if self.manager.type == 'resourcegroup':
resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceGroupName eq '%s'" % resource['name'],
"eventChannels eq 'Operation'",
"resourceType eq '%s'" % resource_type
])
# other Azure resources
else:
resource_type = resource['type']
query_filter = " and ".join([
"eventTimestamp ge '%s'" % start_time,
"resourceUri eq '%s'" % resource['id'],
"eventChannels eq 'Operation'",
"resourceType eq '%s'" % resource_type
])
# fetch activity logs
logs = self.client.activity_logs.list(
filter=query_filter,
select=self.query_select
)
# get the user who issued the first operation
operation_name = "%s/write" % resource_type
first_event = None
for l in logs:
if l.operation_name.value and l.operation_name.value.lower() == operation_name.lower():
first_event = l
resource['c7n:first_iam_event'] = first_event
return first_event
