def _getMetricStatisticsTimeSlice(cls, period):
""" Determine metric statistics collection time range for the maximum range
appropriate for aggregation of the statistic values of an Autostack
:param period: Metric period in seconds; must be multiple of 60
:type period: integer
:returns: time range for collecting metric statistics adjusted for integral
number of metric periods.
:rtype: grok.app.runtime.aggregator_utils.TimeRange
"""
startTime, endTime = cloudwatch_utils.getMetricCollectionTimeRange(startTime=None, endTime=None, period=period)
return TimeRange(start=startTime, end=endTime)
def _getMetricStatisticsTimeSlice(cls, period):
""" Determine metric statistics collection time range for the maximum range
appropriate for aggregation of the statistic values of an Autostack
:param period: Metric period in seconds; must be multiple of 60
:type period: integer
:returns: time range for collecting metric statistics adjusted for integral
number of metric periods.
:rtype: grok.app.runtime.aggregator_utils.TimeRange
"""
startTime, endTime = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=None, endTime=None, period=period)
return TimeRange(start=startTime, end=endTime)
def _getMetricCollectionTimeSlice(cls, startTime, period):
""" Determine metric data collection time range.
:param startTime: UTC start time of planned metric data collection; may be
None when called for the first time, in which case a start
time will be calculated and returned in the result (even
if there is not enough time for at least one period of
metric data collection)
:type startTime: datetime.datetime
:param period: Metric period in seconds; must be multiple of 60
:type period: integer
:returns: time range for collecting metrics adjusted for integral number of
periods. If there is not enough time for at least one period,
then end-time will be set equal to start-time
:rtype: grok.app.runtime.aggregator_utils.TimeRange
"""
startTime, endTime = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=startTime, endTime=None, period=period)
return TimeRange(start=startTime, end=endTime)
def getMetricStatistics(self, start, end):
""" Retrieve metric data statistics for the given time range
:param start: UTC start time of the metric data range. The start value
is inclusive: results include datapoints with the time stamp specified. If
set to None, the implementation will choose the start time automatically
based on Cloudwatch metric data expiration policy (14 days at the time of
this writing)
:type start: datetime.datetime
:param end: UTC end time of the metric data range. The end value is
exclusive; results will include datapoints predating the time stamp
specified. If set to None, will use the current UTC time
:type start: datetime.datetime
:returns: a dictionary with the metric's statistics
:rtype: dict; {"min": <min-value>, "max": <max-value>}
"""
defaultMinVal = self.MIN
defaultMaxVal = self.MAX
start, end = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=start,
endTime=end,
period=self.METRIC_PERIOD)
period = end - start
totalSeconds = int(period.total_seconds())
cloudStats = self._queryCloudWatchMetricStats(period=totalSeconds,
start=start,
end=end,
stats=("Maximum", "Minimum"))
cloudStats = cloudStats[0] if cloudStats else None
minVal, maxVal = self._normalizeMinMax(defaultMinVal, defaultMaxVal,
cloudStats)
self._log.debug("getMetricStatistics for metric %s: minVal=%g, maxVal=%g.",
self.METRIC_NAME, minVal, maxVal)
return {"min": minVal, "max": maxVal}
def getMetricStatistics(self, start, end):
""" Retrieve metric data statistics for the given time range
:param start: UTC start time of the metric data range. The start value
is inclusive: results include datapoints with the time stamp specified. If
set to None, the implementation will choose the start time automatically
based on Cloudwatch metric data expiration policy (14 days at the time of
this writing)
:type start: datetime.datetime
:param end: UTC end time of the metric data range. The end value is
exclusive; results will include datapoints predating the time stamp
specified. If set to None, will use the current UTC time
:type start: datetime.datetime
:returns: a dictionary with the metric's statistics
:rtype: dict; {"min": <min-value>, "max": <max-value>}
"""
defaultMinVal = self.MIN
defaultMaxVal = self.MAX
start, end = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=start, endTime=end, period=self.METRIC_PERIOD)
period = end - start
totalSeconds = int(period.total_seconds())
cloudStats = self._queryCloudWatchMetricStats(period=totalSeconds,
start=start,
end=end,
stats=("Maximum",
"Minimum"))
cloudStats = cloudStats[0] if cloudStats else None
minVal, maxVal = self._normalizeMinMax(defaultMinVal, defaultMaxVal,
cloudStats)
self._log.debug(
"getMetricStatistics for metric %s: minVal=%g, maxVal=%g.",
self.METRIC_NAME, minVal, maxVal)
return {"min": minVal, "max": maxVal}
def _getMetricCollectionTimeSlice(cls, startTime, period):
""" Determine metric data collection time range.
:param startTime: UTC start time of planned metric data collection; may be
None when called for the first time, in which case a start
time will be calculated and returned in the result (even
if there is not enough time for at least one period of
metric data collection)
:type startTime: datetime.datetime
:param period: Metric period in seconds; must be multiple of 60
:type period: integer
:returns: time range for collecting metrics adjusted for integral number of
periods. If there is not enough time for at least one period,
then end-time will be set equal to start-time
:rtype: grok.app.runtime.aggregator_utils.TimeRange
"""
startTime, endTime = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=startTime, endTime=None, period=period
)
return TimeRange(start=startTime, end=endTime)
def getMetricData(self, start, end):
""" Retrieve metric data for the given time range
:param start: UTC start time of the metric data range. The start value
is inclusive: results include datapoints with the time stamp specified. If
set to None, the implementation will choose the start time automatically
based on Cloudwatch metric data expiration policy (14 days at the time of
this writing)
:type start: datetime.datetime
:param end: UTC end time of the metric data range. The end value is
exclusive; results will include datapoints predating the time stamp
specified. If set to None, will use the current UTC time as end
:type end: datetime.datetime
:returns: A two-tuple (<data-sequence>, <next-start-time>).
<data-sequence> is a possibly empty sequence of data points sorted by
timestamp in ascending order. Each data point is a two-tuple of
(<datetime timestamp>, <value>).
<next-start-time> is a datetime.datetime object indicating the UTC start
time to use in next call to this method.
:rtype: tuple
"""
period = self.METRIC_PERIOD
stats = [self.STATISTIC]
samples = []
start, end = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=start,
endTime=end,
period=period)
nextCallStartTime = start
# Calculate the number of records returned by this query
remainingSampleSlots = (end - start).total_seconds() // period
if remainingSampleSlots <= 0:
self._log.warning("The requested date range=[%s..%s] is less than "
"period=%ss; adapter=%r", start, end, period, self)
else:
# AWS limits data access to 1440 records
requestLimit = min(remainingSampleSlots, 1440)
fromDate = start
toDate = fromDate + datetime.timedelta(seconds=period * requestLimit)
# Load data in blocks of up to 1440 records
while not samples and toDate <= end and fromDate < toDate:
nextCallStartTime = fromDate
try:
rawdata = self._queryCloudWatchMetricStats(
period=period,
start=fromDate,
end=toDate,
stats=stats)
except boto.exception.BotoServerError as ex:
if ex.status == 400 and ex.error_code == "Throttling":
# TODO: unit-test
raise grok.app.exceptions.MetricThrottleError(repr(ex))
else:
raise
# AWS limits data access to 1440 records
remainingSampleSlots = remainingSampleSlots - requestLimit
fromDate = toDate
requestLimit = min(remainingSampleSlots, 1440)
toDate = fromDate + datetime.timedelta(seconds=period * requestLimit)
if not rawdata:
continue
# Sort by "Timestamp"
rawdata.sort(key=lambda row: row["Timestamp"])
# Format raw data into data points and append to results
samples.extend((e["Timestamp"], e[self.STATISTIC]) for e in rawdata)
if samples:
nextCallStartTime = samples[-1][0] + datetime.timedelta(seconds=period)
return (samples, nextCallStartTime)
def getMetricData(self, start, end):
""" Retrieve metric data for the given time range
:param start: UTC start time of the metric data range. The start value
is inclusive: results include datapoints with the time stamp specified. If
set to None, the implementation will choose the start time automatically
based on Cloudwatch metric data expiration policy (14 days at the time of
this writing)
:type start: datetime.datetime
:param end: UTC end time of the metric data range. The end value is
exclusive; results will include datapoints predating the time stamp
specified. If set to None, will use the current UTC time as end
:type end: datetime.datetime
:returns: A two-tuple (<data-sequence>, <next-start-time>).
<data-sequence> is a possibly empty sequence of data points sorted by
timestamp in ascending order. Each data point is a two-tuple of
(<datetime timestamp>, <value>).
<next-start-time> is a datetime.datetime object indicating the UTC start
time to use in next call to this method.
:rtype: tuple
"""
period = self.METRIC_PERIOD
stats = [self.STATISTIC]
samples = []
start, end = cloudwatch_utils.getMetricCollectionTimeRange(
startTime=start, endTime=end, period=period)
nextCallStartTime = start
# Calculate the number of records returned by this query
remainingSampleSlots = (end - start).total_seconds() // period
if remainingSampleSlots <= 0:
self._log.warning(
"The requested date range=[%s..%s] is less than "
"period=%ss; adapter=%r", start, end, period, self)
else:
# AWS limits data access to 1440 records
requestLimit = min(remainingSampleSlots, 1440)
fromDate = start
toDate = fromDate + datetime.timedelta(seconds=period *
requestLimit)
# Load data in blocks of up to 1440 records
while not samples and toDate <= end and fromDate < toDate:
nextCallStartTime = fromDate
try:
rawdata = self._queryCloudWatchMetricStats(period=period,
start=fromDate,
end=toDate,
stats=stats)
except boto.exception.BotoServerError as ex:
if ex.status == 400 and ex.error_code == "Throttling":
# TODO: unit-test
raise grok.app.exceptions.MetricThrottleError(repr(ex))
else:
raise
# AWS limits data access to 1440 records
remainingSampleSlots = remainingSampleSlots - requestLimit
fromDate = toDate
requestLimit = min(remainingSampleSlots, 1440)
toDate = fromDate + datetime.timedelta(seconds=period *
requestLimit)
if not rawdata:
continue
# Sort by "Timestamp"
rawdata.sort(key=lambda row: row["Timestamp"])
# Format raw data into data points and append to results
samples.extend(
(e["Timestamp"], e[self.STATISTIC]) for e in rawdata)
if samples:
nextCallStartTime = samples[-1][0] + datetime.timedelta(
seconds=period)
return (samples, nextCallStartTime)
