def _getData(cls, regionName, instanceID, metricName, stats, unit, period,
startTime, endTime):
""" For experimentation """
cw = cloudwatch.connect_to_region(region_name=regionName,
**getAWSCredentials())
data = cw.get_metric_statistics(period=period,
start_time=startTime,
end_time=endTime,
metric_name=metricName,
namespace="AWS/EC2",
statistics=stats,
dimensions=dict(InstanceId=instanceID),
unit=unit)
return data
def _getData(cls, regionName, instanceID, metricName, stats, unit, period, startTime, endTime):
""" For experimentation """
cw = cloudwatch.connect_to_region(region_name=regionName, **getAWSCredentials())
data = cw.get_metric_statistics(
period=period,
start_time=startTime,
end_time=endTime,
metric_name=metricName,
namespace="AWS/EC2",
statistics=stats,
dimensions=dict(InstanceId=instanceID),
unit=unit,
)
return data
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(refID=task.refID,
metricID=task.metricID,
instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0), )
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region,
**getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"],
value=getValue(rawItem, task.stats))
for rawItem in rawdata)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
def _collectMetrics(task):
""" Executed via multiprocessing Pool: Collect metric data
:param task: a _MetricCollectionTask object
:returns: a _MetricCollectionTaskResult object
"""
log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")
taskResult = _MetricCollectionTaskResult(refID=task.refID, metricID=task.metricID, instanceID=task.instanceID)
try:
if task.metricName == "InstanceCount":
timestamp = datetime.utcnow().replace(second=0, microsecond=0)
taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0),)
else:
backoffSec = 0.75
backoffGrowthFactor = 1.5
maxBackoffSec = 5
rawdata = None
while rawdata is None:
try:
cw = cloudwatch.connect_to_region(region_name=task.region, **getAWSCredentials())
rawdata = cw.get_metric_statistics(
period=task.period,
start_time=task.timeRange.start,
end_time=task.timeRange.end,
metric_name=task.metricName,
namespace="AWS/EC2",
statistics=task.stats,
dimensions=dict(InstanceId=task.instanceID),
unit=task.unit,
)
except BotoServerError as e:
if e.status == 400 and e.error_code == "Throttling":
# TODO: unit-test
log.info("Throttling: %r", e)
if backoffSec <= maxBackoffSec:
time.sleep(backoffSec)
backoffSec *= backoffGrowthFactor
else:
raise app_exceptions.MetricThrottleError(repr(e))
else:
raise
# Sort the data by timestamp in ascending order
rawdata.sort(key=lambda row: row["Timestamp"])
# Convert raw data to MetricRecord objects
def getValue(rawDataItem, stats):
if isinstance(stats, basestring):
return rawDataItem[stats]
else:
return dict((field, rawDataItem[field]) for field in stats)
taskResult.data = tuple(
MetricRecord(timestamp=rawItem["Timestamp"], value=getValue(rawItem, task.stats)) for rawItem in rawdata
)
except Exception as e: # pylint: disable=W0703
log.exception("Error in task=%r", task)
taskResult.exception = e
finally:
taskResult.duration = time.time() - taskResult.creationTime
return taskResult
