def validateInterval(value):
try:
parseTimeOffset(value)
except (IndexError, KeyError, TypeError, ValueError) as e:
raise ValueError('Invalid interval value: {value}: {e}'.format(
value=repr(value), e=str(e)))
return value
def timeShift(requestContext, seriesList, timeShift):
"""
Takes one metric or a wildcard seriesList, followed by a length of time,
surrounded by double quotes. (See the URL API for examples of time formats.)
Draw the selected metrics shifted back in time.
Useful for comparing a metric against itself.
Example:
.. code-block:: none
&target=timeShift(Sales.widgets.largeBlue,"7d")
"""
delta = abs( parseTimeOffset(timeShift) )
myContext = requestContext.copy()
myContext['startTime'] = requestContext['startTime'] - delta
myContext['endTime'] = requestContext['endTime'] - delta
series = seriesList[0] # if len(seriesList) > 1, they will all have the same pathExpression, which is all we care about.
results = []
for shiftedSeries in evaluateTarget(myContext, series.pathExpression):
shiftedSeries.name = 'timeShift(%s, %s)' % (shiftedSeries.name, timeShift)
shiftedSeries.start = series.start
shiftedSeries.end = series.end
results.append(shiftedSeries)
return results
def smooth(requestContext, seriesList, intervalString='5min'):
results = []
interval = int(to_seconds(parseTimeOffset(intervalString)))
for series in seriesList:
if series.step < interval:
values_per_point = interval // series.step
series.consolidate(values_per_point)
series.step = interval
results.append(series)
return results
def timeShift(requestContext, seriesList, timeShift):
delta = abs(parseTimeOffset(timeShift))
myContext = requestContext.copy()
myContext['startTime'] = requestContext['startTime'] - delta
myContext['endTime'] = requestContext['endTime'] - delta
series = seriesList[
0] # if len(seriesList) > 1, they will all have the same pathExpression, which is all we care about.
results = []
for shiftedSeries in evaluateTarget(myContext, series.pathExpression):
shiftedSeries.name = 'timeShift(%s, %s)' % (shiftedSeries.name,
timeShift)
shiftedSeries.start = series.start
shiftedSeries.end = series.end
results.append(shiftedSeries)
return results
def summarize(requestContext, seriesList, intervalString):
results = []
delta = parseTimeOffset(intervalString)
interval = delta.seconds + (delta.days * 86400)
for series in seriesList:
buckets = {}
timestamps = range(int(series.start), int(series.end),
int(series.step))
datapoints = zip(timestamps, series)
for (timestamp, value) in datapoints:
bucketInterval = timestamp - (timestamp % interval)
if bucketInterval not in buckets:
buckets[bucketInterval] = []
if value is not None:
buckets[bucketInterval].append(value)
newStart = series.start - (series.start % interval)
newEnd = series.end - (series.end % interval) + interval
newValues = []
for timestamp in range(newStart, newEnd, interval):
bucket = buckets.get(timestamp, [])
if bucket:
newValues.append(sum(bucket))
else:
newValues.append(None)
newName = "summarize(%s, \"%s\")" % (series.name, intervalString)
newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def summarize(requestContext, seriesList, intervalString):
results = []
delta = parseTimeOffset(intervalString)
interval = delta.seconds + (delta.days * 86400)
for series in seriesList:
buckets = {}
timestamps = range( int(series.start), int(series.end), int(series.step) )
datapoints = zip(timestamps, series)
for (timestamp, value) in datapoints:
bucketInterval = timestamp - (timestamp % interval)
if bucketInterval not in buckets:
buckets[bucketInterval] = []
if value is not None:
buckets[bucketInterval].append(value)
newStart = series.start - (series.start % interval)
newEnd = series.end - (series.end % interval) + interval
newValues = []
for timestamp in range(newStart, newEnd, interval):
bucket = buckets.get(timestamp, [])
if bucket:
newValues.append( sum(bucket) )
else:
newValues.append( None )
newName = "summarize(%s, \"%s\")" % (series.name, intervalString)
newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def test_parse_plus_only_returns_zero(self):
time_ref = parseTimeOffset("+")
expected = datetime.timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_digits_only_raises_exception(self):
with self.assertRaises(Exception):
time_ref = parseTimeOffset("10")
def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError(self):
with self.assertRaises(KeyError):
time_ref = parseTimeOffset("Something")
def test_parse_None_returns_empty_timedelta(self):
time_ref = parseTimeOffset(None)
expected = datetime.timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_minus_only_returns_zero(self):
time_ref = parseTimeOffset("-")
expected = timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_one_year_returns_365_days(self):
time_ref = parseTimeOffset("1year")
expected = timedelta(365)
self.assertEquals(time_ref, expected)
def test_parse_two_months_returns_sixty_days(self):
time_ref = parseTimeOffset("2months")
expected = timedelta(60)
self.assertEquals(time_ref, expected)
def test_parse_one_year_returns_365_days(self):
time_ref = parseTimeOffset("1year")
expected = datetime.timedelta(365)
self.assertEquals(time_ref, expected)
def test_parse_five_weeks(self):
time_ref = parseTimeOffset("5weeks")
expected = datetime.timedelta(weeks=5)
self.assertEquals(time_ref, expected)
def test_parse_minus_ten_days(self):
time_ref = parseTimeOffset("-10days")
expected = datetime.timedelta(-10)
self.assertEquals(time_ref, expected)
def ASAP(requestContext, seriesList, resolution=1000):
'''
use the ASAP smoothing on a series
https://arxiv.org/pdf/1703.00983.pdf
https://raw.githubusercontent.com/stanford-futuredata/ASAP/master/ASAP.py
:param requestContext:
:param seriesList:
:param resolution: either number of points to keep or a time resolution
:return: smoothed(seriesList)
'''
if not seriesList:
return []
windowInterval = None
if isinstance(resolution, six.string_types):
delta = parseTimeOffset(resolution)
windowInterval = abs(delta.seconds + (delta.days * 86400))
if windowInterval:
previewSeconds = windowInterval
else:
previewSeconds = max([s.step for s in seriesList]) * int(resolution)
# ignore original data and pull new, including our preview
# data from earlier is needed to calculate the early results
newContext = requestContext.copy()
newContext['startTime'] = (requestContext['startTime'] -
timedelta(seconds=previewSeconds))
previewList = evaluateTokens(newContext, requestContext['args'][0])
result = []
for series in previewList:
if windowInterval:
# the resolution here is really the number of points to maintain
# so we need to convert the "seconds" to num points
windowPoints = round((series.end - series.start) / windowInterval)
else:
use_res = int(resolution)
if len(series) < use_res:
use_res = len(series)
windowPoints = use_res
if isinstance(resolution, six.string_types):
newName = 'asap(%s,"%s")' % (series.name, resolution)
else:
newName = "asap(%s,%s)" % (series.name, resolution)
step_guess = (series.end - series.start) // windowPoints
newSeries = TimeSeries(
newName,
series.start,
series.end,
step_guess,
[]
)
newSeries.pathExpression = newName
# detect "none" lists
if len([v for v in series if v is not None]) <= 1:
newSeries.extend(series)
else:
# the "resolution" is a suggestion,
# the algo will alter it some inorder
# to get the best view for things
new_s = smooth(series, windowPoints)
# steps need to be ints, so we must force the issue
new_step = round((series.end - series.start) / len(new_s))
newSeries.step = new_step
newSeries.extend(new_s)
result.append(newSeries)
return result
def validateInterval(value):
try:
parseTimeOffset(value)
except Exception:
return False
return True
def hitcount(requestContext, seriesList, intervalString):
"""
Estimate hit counts from a list of time series.
This function assumes the values in each time series represent
hits per second. It calculates hits per some larger interval
such as per day or per hour. This function is like summarize(),
except that it compensates automatically for different time scales
(so that a similar graph results from using either fine-grained
or coarse-grained records) and handles rarely-occurring events
gracefully.
"""
results = []
delta = parseTimeOffset(intervalString)
interval = int(delta.seconds + (delta.days * 86400))
for series in seriesList:
length = len(series)
step = int(series.step)
bucket_count = int(math.ceil(float(series.end - series.start) / interval))
buckets = [[] for _ in range(bucket_count)]
newStart = int(series.end - bucket_count * interval)
for i, value in enumerate(series):
if value is None:
continue
start_time = int(series.start + i * step)
start_bucket, start_mod = divmod(start_time - newStart, interval)
end_time = start_time + step
end_bucket, end_mod = divmod(end_time - newStart, interval)
if end_bucket >= bucket_count:
end_bucket = bucket_count - 1
end_mod = interval
if start_bucket == end_bucket:
# All of the hits go to a single bucket.
if start_bucket >= 0:
buckets[start_bucket].append(value * (end_mod - start_mod))
else:
# Spread the hits among 2 or more buckets.
if start_bucket >= 0:
buckets[start_bucket].append(value * (interval - start_mod))
hits_per_bucket = value * interval
for j in range(start_bucket + 1, end_bucket):
buckets[j].append(hits_per_bucket)
if end_mod > 0:
buckets[end_bucket].append(value * end_mod)
newValues = []
for bucket in buckets:
if bucket:
newValues.append( sum(bucket) )
else:
newValues.append(None)
newName = 'hitcount(%s, "%s")' % (series.name, intervalString)
newSeries = TimeSeries(newName, newStart, series.end, interval, newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def test_parse_five_minutes(self):
time_ref = parseTimeOffset("5minutes")
expected = datetime.timedelta(minutes=5)
self.assertEquals(time_ref, expected)
def hitcount(requestContext, seriesList, intervalString):
"""Estimate hit counts from a list of time series.
This function assumes the values in each time series represent
hits per second. It calculates hits per some larger interval
such as per day or per hour. This function is like summarize(),
except that it compensates automatically for different time scales
(so that a similar graph results from using either fine-grained
or coarse-grained records) and handles rarely-occurring events
gracefully.
"""
results = []
delta = parseTimeOffset(intervalString)
interval = int(delta.seconds + (delta.days * 86400))
for series in seriesList:
length = len(series)
step = int(series.step)
bucket_count = int(
math.ceil(float(series.end - series.start) / interval))
buckets = [[] for _ in range(bucket_count)]
newStart = int(series.end - bucket_count * interval)
for i, value in enumerate(series):
if value is None:
continue
start_time = int(series.start + i * step)
start_bucket, start_mod = divmod(start_time - newStart, interval)
end_time = start_time + step
end_bucket, end_mod = divmod(end_time - newStart, interval)
if end_bucket >= bucket_count:
end_bucket = bucket_count - 1
end_mod = interval
if start_bucket == end_bucket:
# All of the hits go to a single bucket.
if start_bucket >= 0:
buckets[start_bucket].append(value * (end_mod - start_mod))
else:
# Spread the hits among 2 or more buckets.
if start_bucket >= 0:
buckets[start_bucket].append(value *
(interval - start_mod))
hits_per_bucket = value * interval
for j in range(start_bucket + 1, end_bucket):
buckets[j].append(hits_per_bucket)
if end_mod > 0:
buckets[end_bucket].append(value * end_mod)
newValues = []
for bucket in buckets:
if bucket:
newValues.append(sum(bucket))
else:
newValues.append(None)
newName = 'hitcount(%s, "%s")' % (series.name, intervalString)
newSeries = TimeSeries(newName, newStart, series.end, interval,
newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def test_parse_two_months_returns_sixty_days(self):
time_ref = parseTimeOffset("2months")
expected = datetime.timedelta(60)
self.assertEquals(time_ref, expected)
def test_parse_None_returns_empty_timedelta(self):
time_ref = parseTimeOffset(None)
expected = timedelta(0)
self.assertEquals(time_ref, expected)
def summarize(requestContext, seriesList, intervalString, func='sum', alignToFrom=False):
"""
Summarize the data into interval buckets of a certain size.
By default, the contents of each interval bucket are summed together. This is
useful for counters where each increment represents a discrete event and
retrieving a "per X" value requires summing all the events in that interval.
Specifying 'avg' instead will return the mean for each bucket, which can be more
useful when the value is a gauge that represents a certain value in time.
'max', 'min' or 'last' can also be specified.
By default, buckets are caculated by rounding to the nearest interval. This
works well for intervals smaller than a day. For example, 22:32 will end up
in the bucket 22:00-23:00 when the interval=1hour.
Passing alignToFrom=true will instead create buckets starting at the from
time. In this case, the bucket for 22:32 depends on the from time. If
from=6:30 then the 1hour bucket for 22:32 is 22:30-23:30.
Example:
.. code-block:: none
&target=summarize(counter.errors, "1hour") # total errors per hour
&target=summarize(nonNegativeDerivative(gauge.num_users), "1week") # new users per week
&target=summarize(queue.size, "1hour", "avg") # average queue size per hour
&target=summarize(queue.size, "1hour", "max") # maximum queue size during each hour
&target=summarize(metric, "13week", "avg", true)&from=midnight+20100101 # 2010 Q1-4
"""
results = []
delta = parseTimeOffset(intervalString)
interval = delta.seconds + (delta.days * 86400)
for series in seriesList:
buckets = {}
timestamps = range( int(series.start), int(series.end), int(series.step) )
datapoints = zip(timestamps, series)
for (timestamp, value) in datapoints:
if alignToFrom:
bucketInterval = int((timestamp - series.start) / interval)
else:
bucketInterval = timestamp - (timestamp % interval)
if bucketInterval not in buckets:
buckets[bucketInterval] = []
if value is not None:
buckets[bucketInterval].append(value)
if alignToFrom:
newStart = series.start
newEnd = series.end
else:
newStart = series.start - (series.start % interval)
newEnd = series.end - (series.end % interval) + interval
newValues = []
for timestamp in range(newStart, newEnd, interval):
if alignToFrom:
newEnd = timestamp
bucketInterval = int((timestamp - series.start) / interval)
else:
bucketInterval = timestamp - (timestamp % interval)
bucket = buckets.get(bucketInterval, [])
if bucket:
if func == 'avg':
newValues.append( float(sum(bucket)) / float(len(bucket)) )
elif func == 'last':
newValues.append( bucket[len(bucket)-1] )
elif func == 'max':
newValues.append( max(bucket) )
elif func == 'min':
newValues.append( min(bucket) )
else:
newValues.append( sum(bucket) )
else:
newValues.append( None )
if alignToFrom:
newEnd += interval
newName = "summarize(%s, \"%s\")" % (series.name, intervalString)
newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues)
newSeries.pathExpression = newName
results.append(newSeries)
return results
def test_parse_integer_raises_TypeError(self):
with self.assertRaises(TypeError):
time_ref = parseTimeOffset(1)
def test_parse_twelve_months_returns_360_days(self):
time_ref = parseTimeOffset("12months")
expected = timedelta(360)
self.assertEquals(time_ref, expected)
def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError(
self):
with self.assertRaises(KeyError):
time_ref = parseTimeOffset("Something")
def test_parse_two_years_returns_730_days(self):
time_ref = parseTimeOffset("2years")
expected = timedelta(730)
self.assertEquals(time_ref, expected)
def test_parse_m_as_unit_raises_Exception(self):
with self.assertRaises(Exception):
time_ref = parseTimeOffset("1m")
def test_parse_ten_days(self):
time_ref = parseTimeOffset("10days")
expected = timedelta(10)
self.assertEquals(time_ref, expected)
def test_parse_digits_only_raises_exception(self):
with self.assertRaises(Exception):
time_ref = parseTimeOffset("10")
def test_parse_integer_raises_TypeError(self):
with self.assertRaises(TypeError):
time_ref = parseTimeOffset(1)
def test_parse_alpha_only_raises_KeyError(self):
with self.assertRaises(KeyError):
time_ref = parseTimeOffset("month")
def test_parse_m_as_unit_raises_Exception(self):
with self.assertRaises(Exception):
time_ref = parseTimeOffset("1m")
def test_parse_plus_only_returns_zero(self):
time_ref = parseTimeOffset("+")
expected = timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_alpha_only_raises_KeyError(self):
with self.assertRaises(KeyError):
time_ref = parseTimeOffset("month")
def test_parse_zero_days(self):
time_ref = parseTimeOffset("0days")
expected = timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_zero_days(self):
time_ref = parseTimeOffset("0days")
expected = datetime.timedelta(0)
self.assertEquals(time_ref, expected)
def test_parse_minus_ten_days(self):
time_ref = parseTimeOffset("-10days")
expected = timedelta(-10)
self.assertEquals(time_ref, expected)
def test_parse_five_seconds(self):
time_ref = parseTimeOffset("5seconds")
expected = datetime.timedelta(seconds=5)
self.assertEquals(time_ref, expected)
def test_parse_five_seconds(self):
time_ref = parseTimeOffset("5seconds")
expected = timedelta(seconds=5)
self.assertEquals(time_ref, expected)
def test_parse_five_hours(self):
time_ref = parseTimeOffset("5hours")
expected = datetime.timedelta(hours=5)
self.assertEquals(time_ref, expected)
def test_parse_five_minutes(self):
time_ref = parseTimeOffset("5minutes")
expected = timedelta(minutes=5)
self.assertEquals(time_ref, expected)
def test_parse_one_month_returns_thirty_days(self):
time_ref = parseTimeOffset("1month")
expected = datetime.timedelta(30)
self.assertEquals(time_ref, expected)
def test_parse_five_hours(self):
time_ref = parseTimeOffset("5hours")
expected = timedelta(hours=5)
self.assertEquals(time_ref, expected)
def test_parse_twelve_months_returns_360_days(self):
time_ref = parseTimeOffset("12months")
expected = datetime.timedelta(360)
self.assertEquals(time_ref, expected)
def test_parse_five_weeks(self):
time_ref = parseTimeOffset("5weeks")
expected = timedelta(weeks=5)
self.assertEquals(time_ref, expected)
def test_parse_two_years_returns_730_days(self):
time_ref = parseTimeOffset("2years")
expected = datetime.timedelta(730)
self.assertEquals(time_ref, expected)
def test_parse_one_month_returns_thirty_days(self):
time_ref = parseTimeOffset("1month")
expected = timedelta(30)
self.assertEquals(time_ref, expected)
