def _enumerate_resources(self, service, service_name, region, account,
resource_re):
all_resources = skew.resources.all_types('aws', service_name)
LOG.debug('account: %s', account)
LOG.debug('all_resources: %s', all_resources)
if '/' in resource_re:
resource_type, resource_id = resource_re.split('/', 1)
elif ':' in resource_re:
resource_type, resource_id = resource_re.split(':', 1)
else:
resource_type = resource_re
resource_id = None
resource_matcher = Matcher(all_resources, resource_type)
endpoint = Endpoint(service, region, account)
for resource_type in resource_matcher:
kwargs = {}
resource_path = '.'.join(['aws', service_name, resource_type])
resource_cls = skew.resources.find_resource_class(resource_path)
do_client_side_filtering = False
if resource_id and resource_id != '*':
# If we are looking for a specific resource and the
# API provides a way to filter on a specific resource
# id then let's insert the right parameter to do the filtering.
# If the API does not support that, we will have to filter
# after we get all of the results.
filter_name = resource_cls.Meta.filter_name
if filter_name:
if resource_cls.Meta.filter_type == 'list':
kwargs[filter_name] = [resource_id]
else:
kwargs[filter_name] = resource_id
else:
do_client_side_filtering = True
enum_op, path = resource_cls.Meta.enum_spec
data = endpoint.call(enum_op, query=path, **kwargs)
for d in data:
LOG.debug(d)
if do_client_side_filtering:
# If the API does not support filtering, the resource
# class should provide a filter method that will
# return True if the returned data matches the
# resource ID we are looking for.
if not resource_cls.filter(resource_id, d):
continue
resource = resource_cls(endpoint, d)
self._fire_event('resource-create',
self._groups['provider'],
service_name,
region,
account,
resource_type,
resource.id,
resource=resource)
yield resource
def enumerate(self, values):
_, provider, service_name, region, account = values
service = self._arn.session.get_service(service_name)
endpoint = Endpoint(service, region, account)
if '/' in self.pattern:
resource_type, resource_id = self.pattern.split('/', 1)
elif ':' in self.pattern:
resource_type, resource_id = self.pattern.split(':', 1)
else:
resource_type = self.pattern
resource_id = None
LOG.debug('resource_type=%s, resource_id=%s',
resource_type, resource_id)
for resource_type in self.matches:
kwargs = {}
resource_path = '.'.join(['aws', service_name, resource_type])
resource_cls = skew.resources.find_resource_class(resource_path)
do_client_side_filtering = False
if resource_id and resource_id != '*':
# If we are looking for a specific resource and the
# API provides a way to filter on a specific resource
# id then let's insert the right parameter to do the filtering.
# If the API does not support that, we will have to filter
# after we get all of the results.
filter_name = resource_cls.Meta.filter_name
if filter_name:
if resource_cls.Meta.filter_type == 'list':
kwargs[filter_name] = [resource_id]
else:
kwargs[filter_name] = resource_id
else:
do_client_side_filtering = True
enum_op, path = resource_cls.Meta.enum_spec
data = endpoint.call(enum_op, query=path, **kwargs)
LOG.debug(data)
for d in data:
if do_client_side_filtering:
# If the API does not support filtering, the resource
# class should provide a filter method that will
# return True if the returned data matches the
# resource ID we are looking for.
if not resource_cls.filter(resource_id, d):
continue
resource = resource_cls(endpoint, d, self._arn.query)
self._arn.fire_event(
'resource-create', self._arn.provider,
service_name, region, account,
resource_type, resource.id, resource=resource)
yield resource
def _enumerate_resources(self, service, service_name, region,
account, resource_re):
all_resources = skew.resources.all_types('aws', service_name)
LOG.debug('account: %s', account)
LOG.debug('all_resources: %s', all_resources)
if '/' in resource_re:
resource_type, resource_id = resource_re.split('/', 1)
elif ':' in resource_re:
resource_type, resource_id = resource_re.split(':', 1)
else:
resource_type = resource_re
resource_id = None
resource_matcher = Matcher(all_resources, resource_type)
endpoint = Endpoint(service, region, account)
for resource_type in resource_matcher:
kwargs = {}
resource_path = '.'.join(['aws', service_name, resource_type])
resource_cls = skew.resources.find_resource_class(resource_path)
do_client_side_filtering = False
if resource_id and resource_id != '*':
# If we are looking for a specific resource and the
# API provides a way to filter on a specific resource
# id then let's insert the right parameter to do the filtering.
# If the API does not support that, we will have to filter
# after we get all of the results.
filter_name = resource_cls.Meta.filter_name
if filter_name:
if resource_cls.Meta.filter_type == 'list':
kwargs[filter_name] = [resource_id]
else:
kwargs[filter_name] = resource_id
else:
do_client_side_filtering = True
enum_op, path = resource_cls.Meta.enum_spec
data = endpoint.call(enum_op, query=path, **kwargs)
for d in data:
LOG.debug(d)
if do_client_side_filtering:
# If the API does not support filtering, the resource
# class should provide a filter method that will
# return True if the returned data matches the
# resource ID we are looking for.
if not resource_cls.filter(resource_id, d):
continue
resource = resource_cls(endpoint, d)
self._fire_event('resource-create', self._groups['provider'],
service_name, region, account,
resource_type, resource.id, resource=resource)
yield resource
def __init__(self, endpoint, data, query=None):
self._endpoint = endpoint
self._region = endpoint.region
self._account = endpoint.account
self._query = query
if data is None:
data = {}
self.data = data
if self._query:
self.filtered_data = self._query.search(self.data)
else:
self.filtered_data = None
if hasattr(self.Meta, 'id') and isinstance(self.data, dict):
self._id = self.data.get(self.Meta.id, '')
else:
self._id = ''
self._cloudwatch = None
if hasattr(self.Meta, 'dimension') and self.Meta.dimension:
cloudwatch = self._endpoint.service.session.get_service(
'cloudwatch')
self._cloudwatch = Endpoint(
cloudwatch, self._region, self._account)
self._metrics = None
self._name = None
self._date = None
self._tags = None
def __init__(self, endpoint, data):
self._endpoint = endpoint
self._region = endpoint.region
self._account = endpoint.account
if data is None:
data = {}
self.data = data
if hasattr(self.Meta, 'id') and isinstance(self.data, dict):
self._id = self.data.get(self.Meta.id, '')
else:
self._id = ''
self._cloudwatch = None
if hasattr(self.Meta, 'dimension') and self.Meta.dimension:
cloudwatch = self._endpoint.service.session.get_service(
'cloudwatch')
self._cloudwatch = Endpoint(cloudwatch, self._region,
self._account)
self._metrics = None
self._name = None
self._date = None
self._tags = None
def resource_from_arn(arn, data):
session = botocore.session.get_session()
parts = ArnComponents(*arn.split(':', 6))
service = session.get_service(parts.service)
if ':' in parts.resource:
resource_type, resource_id = parts.resource.split(':')
elif '/' in parts.resource:
resource_type, resource_id = parts.resource.split('/')
else:
resource_type = parts.resource
endpoint = Endpoint(service, parts.region, parts.account)
resource_path = '.'.join(['aws', parts.service, resource_type])
resource_cls = find_resource_class(resource_path)
return resource_cls(endpoint, data)
class AWSResource(Resource):
"""
Each Resource class defines a Config variable at the class level. This
is a dictionary that gives the specifics about which service the resource
belongs to and how to enumerate the resource.
Each entry in the dictionary we define:
* service - The AWS service in which this resource is defined.
* enum_spec - The enumeration configuration. This is a tuple consisting
of the name of the operation to call to enumerate the resources and
a jmespath query that will be run against the result of the operation
to retrieve the list of resources.
* tags_spec - Some AWS resources return the tags for the resource in
the enumeration operation (e.g. DescribeInstances) while others
require a second operation to retrieve the tags. If a second
operation is required the ``tags_spec`` describes how to make the
call. It is a tuple consisting of:
* operation name
* jmespath query to find the tags in the response
* the name of the parameter to send to identify the specific resource
(this is not always the same as filter_name, e.g. RDS)
* the value of the parameter to send to identify the specific resource
(this is not always the same as the id, e.g. RDS)
* detail_spec - Some services provide only summary information in the
list or describe method and require you to make another request to get
the detailed info for a specific resource. If that is the case, this
would contain a tuple consisting of the operation to call for the
details, the parameter name to pass in to identify the desired
resource and the jmespath filter to apply to the results to get
the details.
* id - The name of the field within the resource data that uniquely
identifies the resource.
* dimension - The CloudWatch dimension for this resource. A value
of None indicates that this resource is not monitored by CloudWatch.
* filter_name - By default, the enumerator returns all resources of a
given type. But you can also tell it to filter the results by
passing in a list of id's. This parameter tells it the name of the
parameter to use to specify this list of id's.
"""
class Meta(object):
type = 'awsresource'
@classmethod
def filter(cls, resource_id, data):
pass
def __init__(self, endpoint, data):
self._endpoint = endpoint
self._region = endpoint.region
self._account = endpoint.account
if data is None:
data = {}
self.data = data
if hasattr(self.Meta, 'id') and isinstance(self.data, dict):
self._id = self.data.get(self.Meta.id, '')
else:
self._id = ''
self._cloudwatch = None
if hasattr(self.Meta, 'dimension') and self.Meta.dimension:
cloudwatch = self._endpoint.service.session.get_service(
'cloudwatch')
self._cloudwatch = Endpoint(
cloudwatch, self._region, self._account)
self._metrics = None
self._name = None
self._date = None
self._tags = None
def __repr__(self):
return self.arn
@property
def arn(self):
return 'arn:aws:%s:%s:%s:%s/%s' % (
self._endpoint.service.endpoint_prefix,
self._region, self._account, self.resourcetype, self.id)
@property
def metrics(self):
if self._metrics is None:
if self._cloudwatch:
data = self._cloudwatch.call(
'ListMetrics',
dimensions=[{'Name': self.Meta.dimension,
'Value': self._id}])
self._metrics = jmespath.search('Metrics', data)
else:
self._metrics = []
return self._metrics
@property
def tags(self):
"""
Convert the ugly Tags JSON into a real dictionary and
memorize the result.
"""
if self._tags is None:
LOG.debug('need to build tags')
self._tags = {}
if hasattr(self.Meta, 'tags_spec'):
LOG.debug('have a tags_spec')
method, path, param_name, param_value = self.Meta.tags_spec
kwargs = {}
filter_type = getattr(self.Meta, 'filter_type', None)
if filter_type == 'list':
kwargs = {param_name: [getattr(self, param_value)]}
else:
kwargs = {param_name: getattr(self, param_value)}
LOG.debug('fetching tags')
self.data['Tags'] = self._endpoint.call(
method, query=path, **kwargs)
LOG.debug(self.data['Tags'])
if 'Tags' in self.data:
for kvpair in self.data['Tags']:
if kvpair['Key'] in self._tags:
if not isinstance(self._tags[kvpair['Key']], list):
self._tags[kvpair['Key']] = [self._tags[kvpair['Key']]]
self._tags[kvpair['Key'].append(kvpair['Value'])]
else:
self._tags[kvpair['Key']] = kvpair['Value']
return self._tags
def find_metric(self, metric_name):
for m in self.metrics:
if m['MetricName'] == metric_name:
return m
return None
def _total_seconds(self, delta):
# python2.6 does not have timedelta.total_seconds() so we have
# to calculate this ourselves. This is straight from the
# datetime docs.
return ((delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
* 10 ** 6) / 10 ** 6)
def get_metric_data(self, metric_name=None, metric=None,
days=None, hours=1, minutes=None,
statistics=None, period=None):
"""
Get metric data for this resource. You can specify the time
frame for the data as either the number of days or number of
hours. The maximum window is 14 days. Based on the time frame
this method will calculate the correct ``period`` to return
the maximum number of data points up to the CloudWatch max
of 1440.
:type metric_name: str
:param metric_name: The name of the metric this data will
pertain to.
:type days: int
:param days: The number of days worth of data to return.
You can specify either ``days`` or ``hours``. The default
is one hour. The maximum value is 14 days.
:type hours: int
:param hours: The number of hours worth of data to return.
You can specify either ``days`` or ``hours``. The default
is one hour. The maximum value is 14 days.
:type statistics: list of str
:param statistics: The metric statistics to return. The default
value is **Average**. Possible values are:
* Average
* Sum
* SampleCount
* Maximum
* Minimum
:returns: A ``MetricData`` object that contains both the CloudWatch
data as well as the ``period`` used since this value may have
been calculated by skew.
"""
if not statistics:
statistics = ['Average']
if days:
delta = datetime.timedelta(days=days)
elif hours:
delta = datetime.timedelta(hours=hours)
else:
delta = datetime.timedelta(minutes=minutes)
if not period:
period = max(60, self._total_seconds(delta) // 1440)
if not metric:
metric = self.find_metric(metric_name)
if metric and self._cloudwatch:
end = datetime.datetime.utcnow()
start = end - delta
data = self._cloudwatch.call(
'GetMetricStatistics',
dimensions=metric['Dimensions'],
namespace=metric['Namespace'],
metric_name=metric['MetricName'],
start_time=start.isoformat(), end_time=end.isoformat(),
statistics=statistics, period=period)
return MetricData(jmespath.search('Datapoints', data),
period)
else:
raise ValueError('Metric (%s) not available' % metric_name)
class AWSResource(Resource):
"""
Each Resource class defines a Config variable at the class level. This
is a dictionary that gives the specifics about which service the resource
belongs to and how to enumerate the resource.
Each entry in the dictionary we define:
* service - The AWS service in which this resource is defined.
* enum_spec - The enumeration configuration. This is a tuple consisting
of the name of the operation to call to enumerate the resources and
a jmespath query that will be run against the result of the operation
to retrieve the list of resources.
* detail_spec - Some services provide only summary information in the
list or describe method and require you to make another request to get
the detailed info for a specific resource. If that is the case, this
would contain a tuple consisting of the operation to call for the
details, the parameter name to pass in to identify the desired
resource and the jmespath filter to apply to the results to get
the details.
* id - The name of the field within the resource data that uniquely
identifies the resource.
* dimension - The CloudWatch dimension for this resource. A value
of None indicates that this resource is not monitored by CloudWatch.
* filter_name - By default, the enumerator returns all resources of a
given type. But you can also tell it to filter the results by
passing in a list of id's. This parameter tells it the name of the
parameter to use to specify this list of id's.
"""
class Meta(object):
type = 'awsresource'
@classmethod
def filter(cls, resource_id, data):
pass
def __init__(self, endpoint, data):
self._endpoint = endpoint
self._region = endpoint.region
self._account = endpoint.account
if data is None:
data = {}
self.data = data
if hasattr(self.Meta, 'id') and isinstance(self.data, dict):
self._id = self.data.get(self.Meta.id, '')
else:
self._id = ''
self._cloudwatch = None
if hasattr(self.Meta, 'dimension') and self.Meta.dimension:
cloudwatch = self._endpoint.service.session.get_service(
'cloudwatch')
self._cloudwatch = Endpoint(cloudwatch, self._region,
self._account)
self._metrics = None
self._name = None
self._date = None
self._tags = None
def __repr__(self):
return self.arn
@property
def arn(self):
return 'arn:aws:%s:%s:%s:%s/%s' % (
self._endpoint.service.endpoint_prefix, self._region,
self._account, self.resourcetype, self.id)
@property
def metrics(self):
if self._metrics is None:
if self._cloudwatch:
data = self._cloudwatch.call('ListMetrics',
dimensions=[{
'Name': self.Meta.dimension,
'Value': self._id
}])
self._metrics = jmespath.search('Metrics', data)
else:
self._metrics = []
return self._metrics
@property
def tags(self):
"""
Convert the ugly Tags JSON into a real dictionary and
memoize the result.
"""
if self._tags is None:
self._tags = {}
if 'Tags' in self.data:
for kvpair in self.data['Tags']:
if kvpair['Key'] in self._tags:
if not isinstance(self._tags[kvpair['Key']], list):
self._tags[kvpair['Key']] = [
self._tags[kvpair['Key']]
]
self._tags[kvpair['Key'].append(kvpair['Value'])]
else:
self._tags[kvpair['Key']] = kvpair['Value']
return self._tags
def find_metric(self, metric_name):
for m in self.metrics:
if m['MetricName'] == metric_name:
return m
return None
def _total_seconds(self, delta):
# python2.6 does not have timedelta.total_seconds() so we have
# to calculate this ourselves. This is straight from the
# datetime docs.
return ((delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6)
def get_metric_data(self,
metric_name=None,
metric=None,
days=None,
hours=1,
minutes=None,
statistics=None,
period=None):
"""
Get metric data for this resource. You can specify the time
frame for the data as either the number of days or number of
hours. The maximum window is 14 days. Based on the time frame
this method will calculate the correct ``period`` to return
the maximum number of data points up to the CloudWatch max
of 1440.
:type metric_name: str
:param metric_name: The name of the metric this data will
pertain to.
:type days: int
:param days: The number of days worth of data to return.
You can specify either ``days`` or ``hours``. The default
is one hour. The maximum value is 14 days.
:type hours: int
:param hours: The number of hours worth of data to return.
You can specify either ``days`` or ``hours``. The default
is one hour. The maximum value is 14 days.
:type statistics: list of str
:param statistics: The metric statistics to return. The default
value is **Average**. Possible values are:
* Average
* Sum
* SampleCount
* Maximum
* Minimum
"""
if not statistics:
statistics = ['Average']
if days:
delta = datetime.timedelta(days=days)
elif hours:
delta = datetime.timedelta(hours=hours)
else:
delta = datetime.timedelta(minutes=minutes)
if not period:
period = max(60, self._total_seconds(delta) // 1440)
if not metric:
metric = self.find_metric(metric_name)
if metric and self._cloudwatch:
end = datetime.datetime.utcnow()
start = end - delta
data = self._cloudwatch.call('GetMetricStatistics',
dimensions=metric['Dimensions'],
namespace=metric['Namespace'],
metric_name=metric['MetricName'],
start_time=start.isoformat(),
end_time=end.isoformat(),
statistics=statistics,
period=period)
return jmespath.search('Datapoints', data)
else:
raise ValueError('Metric (%s) not available' % metric_name)
