def validateInterval(value): try: parseTimeOffset(value) except (IndexError, KeyError, TypeError, ValueError) as e: raise ValueError('Invalid interval value: {value}: {e}'.format( value=repr(value), e=str(e))) return value
def timeShift(requestContext, seriesList, timeShift): """ Takes one metric or a wildcard seriesList, followed by a length of time, surrounded by double quotes. (See the URL API for examples of time formats.) Draw the selected metrics shifted back in time. Useful for comparing a metric against itself. Example: .. code-block:: none &target=timeShift(Sales.widgets.largeBlue,"7d") """ delta = abs( parseTimeOffset(timeShift) ) myContext = requestContext.copy() myContext['startTime'] = requestContext['startTime'] - delta myContext['endTime'] = requestContext['endTime'] - delta series = seriesList[0] # if len(seriesList) > 1, they will all have the same pathExpression, which is all we care about. results = [] for shiftedSeries in evaluateTarget(myContext, series.pathExpression): shiftedSeries.name = 'timeShift(%s, %s)' % (shiftedSeries.name, timeShift) shiftedSeries.start = series.start shiftedSeries.end = series.end results.append(shiftedSeries) return results
def smooth(requestContext, seriesList, intervalString='5min'): results = [] interval = int(to_seconds(parseTimeOffset(intervalString))) for series in seriesList: if series.step < interval: values_per_point = interval // series.step series.consolidate(values_per_point) series.step = interval results.append(series) return results
def timeShift(requestContext, seriesList, timeShift): delta = abs(parseTimeOffset(timeShift)) myContext = requestContext.copy() myContext['startTime'] = requestContext['startTime'] - delta myContext['endTime'] = requestContext['endTime'] - delta series = seriesList[ 0] # if len(seriesList) > 1, they will all have the same pathExpression, which is all we care about. results = [] for shiftedSeries in evaluateTarget(myContext, series.pathExpression): shiftedSeries.name = 'timeShift(%s, %s)' % (shiftedSeries.name, timeShift) shiftedSeries.start = series.start shiftedSeries.end = series.end results.append(shiftedSeries) return results
def summarize(requestContext, seriesList, intervalString): results = [] delta = parseTimeOffset(intervalString) interval = delta.seconds + (delta.days * 86400) for series in seriesList: buckets = {} timestamps = range(int(series.start), int(series.end), int(series.step)) datapoints = zip(timestamps, series) for (timestamp, value) in datapoints: bucketInterval = timestamp - (timestamp % interval) if bucketInterval not in buckets: buckets[bucketInterval] = [] if value is not None: buckets[bucketInterval].append(value) newStart = series.start - (series.start % interval) newEnd = series.end - (series.end % interval) + interval newValues = [] for timestamp in range(newStart, newEnd, interval): bucket = buckets.get(timestamp, []) if bucket: newValues.append(sum(bucket)) else: newValues.append(None) newName = "summarize(%s, \"%s\")" % (series.name, intervalString) newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) return results
def summarize(requestContext, seriesList, intervalString): results = [] delta = parseTimeOffset(intervalString) interval = delta.seconds + (delta.days * 86400) for series in seriesList: buckets = {} timestamps = range( int(series.start), int(series.end), int(series.step) ) datapoints = zip(timestamps, series) for (timestamp, value) in datapoints: bucketInterval = timestamp - (timestamp % interval) if bucketInterval not in buckets: buckets[bucketInterval] = [] if value is not None: buckets[bucketInterval].append(value) newStart = series.start - (series.start % interval) newEnd = series.end - (series.end % interval) + interval newValues = [] for timestamp in range(newStart, newEnd, interval): bucket = buckets.get(timestamp, []) if bucket: newValues.append( sum(bucket) ) else: newValues.append( None ) newName = "summarize(%s, \"%s\")" % (series.name, intervalString) newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) return results
def test_parse_plus_only_returns_zero(self): time_ref = parseTimeOffset("+") expected = datetime.timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_digits_only_raises_exception(self): with self.assertRaises(Exception): time_ref = parseTimeOffset("10")
def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError(self): with self.assertRaises(KeyError): time_ref = parseTimeOffset("Something")
def test_parse_None_returns_empty_timedelta(self): time_ref = parseTimeOffset(None) expected = datetime.timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_minus_only_returns_zero(self): time_ref = parseTimeOffset("-") expected = timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_one_year_returns_365_days(self): time_ref = parseTimeOffset("1year") expected = timedelta(365) self.assertEquals(time_ref, expected)
def test_parse_two_months_returns_sixty_days(self): time_ref = parseTimeOffset("2months") expected = timedelta(60) self.assertEquals(time_ref, expected)
def test_parse_one_year_returns_365_days(self): time_ref = parseTimeOffset("1year") expected = datetime.timedelta(365) self.assertEquals(time_ref, expected)
def test_parse_five_weeks(self): time_ref = parseTimeOffset("5weeks") expected = datetime.timedelta(weeks=5) self.assertEquals(time_ref, expected)
def test_parse_minus_ten_days(self): time_ref = parseTimeOffset("-10days") expected = datetime.timedelta(-10) self.assertEquals(time_ref, expected)
def ASAP(requestContext, seriesList, resolution=1000): ''' use the ASAP smoothing on a series https://arxiv.org/pdf/1703.00983.pdf https://raw.githubusercontent.com/stanford-futuredata/ASAP/master/ASAP.py :param requestContext: :param seriesList: :param resolution: either number of points to keep or a time resolution :return: smoothed(seriesList) ''' if not seriesList: return [] windowInterval = None if isinstance(resolution, six.string_types): delta = parseTimeOffset(resolution) windowInterval = abs(delta.seconds + (delta.days * 86400)) if windowInterval: previewSeconds = windowInterval else: previewSeconds = max([s.step for s in seriesList]) * int(resolution) # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() newContext['startTime'] = (requestContext['startTime'] - timedelta(seconds=previewSeconds)) previewList = evaluateTokens(newContext, requestContext['args'][0]) result = [] for series in previewList: if windowInterval: # the resolution here is really the number of points to maintain # so we need to convert the "seconds" to num points windowPoints = round((series.end - series.start) / windowInterval) else: use_res = int(resolution) if len(series) < use_res: use_res = len(series) windowPoints = use_res if isinstance(resolution, six.string_types): newName = 'asap(%s,"%s")' % (series.name, resolution) else: newName = "asap(%s,%s)" % (series.name, resolution) step_guess = (series.end - series.start) // windowPoints newSeries = TimeSeries( newName, series.start, series.end, step_guess, [] ) newSeries.pathExpression = newName # detect "none" lists if len([v for v in series if v is not None]) <= 1: newSeries.extend(series) else: # the "resolution" is a suggestion, # the algo will alter it some inorder # to get the best view for things new_s = smooth(series, windowPoints) # steps need to be ints, so we must force the issue new_step = round((series.end - series.start) / len(new_s)) newSeries.step = new_step newSeries.extend(new_s) result.append(newSeries) return result
def validateInterval(value): try: parseTimeOffset(value) except Exception: return False return True
def hitcount(requestContext, seriesList, intervalString): """ Estimate hit counts from a list of time series. This function assumes the values in each time series represent hits per second. It calculates hits per some larger interval such as per day or per hour. This function is like summarize(), except that it compensates automatically for different time scales (so that a similar graph results from using either fine-grained or coarse-grained records) and handles rarely-occurring events gracefully. """ results = [] delta = parseTimeOffset(intervalString) interval = int(delta.seconds + (delta.days * 86400)) for series in seriesList: length = len(series) step = int(series.step) bucket_count = int(math.ceil(float(series.end - series.start) / interval)) buckets = [[] for _ in range(bucket_count)] newStart = int(series.end - bucket_count * interval) for i, value in enumerate(series): if value is None: continue start_time = int(series.start + i * step) start_bucket, start_mod = divmod(start_time - newStart, interval) end_time = start_time + step end_bucket, end_mod = divmod(end_time - newStart, interval) if end_bucket >= bucket_count: end_bucket = bucket_count - 1 end_mod = interval if start_bucket == end_bucket: # All of the hits go to a single bucket. if start_bucket >= 0: buckets[start_bucket].append(value * (end_mod - start_mod)) else: # Spread the hits among 2 or more buckets. if start_bucket >= 0: buckets[start_bucket].append(value * (interval - start_mod)) hits_per_bucket = value * interval for j in range(start_bucket + 1, end_bucket): buckets[j].append(hits_per_bucket) if end_mod > 0: buckets[end_bucket].append(value * end_mod) newValues = [] for bucket in buckets: if bucket: newValues.append( sum(bucket) ) else: newValues.append(None) newName = 'hitcount(%s, "%s")' % (series.name, intervalString) newSeries = TimeSeries(newName, newStart, series.end, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) return results
def test_parse_five_minutes(self): time_ref = parseTimeOffset("5minutes") expected = datetime.timedelta(minutes=5) self.assertEquals(time_ref, expected)
def hitcount(requestContext, seriesList, intervalString): """Estimate hit counts from a list of time series. This function assumes the values in each time series represent hits per second. It calculates hits per some larger interval such as per day or per hour. This function is like summarize(), except that it compensates automatically for different time scales (so that a similar graph results from using either fine-grained or coarse-grained records) and handles rarely-occurring events gracefully. """ results = [] delta = parseTimeOffset(intervalString) interval = int(delta.seconds + (delta.days * 86400)) for series in seriesList: length = len(series) step = int(series.step) bucket_count = int( math.ceil(float(series.end - series.start) / interval)) buckets = [[] for _ in range(bucket_count)] newStart = int(series.end - bucket_count * interval) for i, value in enumerate(series): if value is None: continue start_time = int(series.start + i * step) start_bucket, start_mod = divmod(start_time - newStart, interval) end_time = start_time + step end_bucket, end_mod = divmod(end_time - newStart, interval) if end_bucket >= bucket_count: end_bucket = bucket_count - 1 end_mod = interval if start_bucket == end_bucket: # All of the hits go to a single bucket. if start_bucket >= 0: buckets[start_bucket].append(value * (end_mod - start_mod)) else: # Spread the hits among 2 or more buckets. if start_bucket >= 0: buckets[start_bucket].append(value * (interval - start_mod)) hits_per_bucket = value * interval for j in range(start_bucket + 1, end_bucket): buckets[j].append(hits_per_bucket) if end_mod > 0: buckets[end_bucket].append(value * end_mod) newValues = [] for bucket in buckets: if bucket: newValues.append(sum(bucket)) else: newValues.append(None) newName = 'hitcount(%s, "%s")' % (series.name, intervalString) newSeries = TimeSeries(newName, newStart, series.end, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) return results
def test_parse_two_months_returns_sixty_days(self): time_ref = parseTimeOffset("2months") expected = datetime.timedelta(60) self.assertEquals(time_ref, expected)
def test_parse_None_returns_empty_timedelta(self): time_ref = parseTimeOffset(None) expected = timedelta(0) self.assertEquals(time_ref, expected)
def summarize(requestContext, seriesList, intervalString, func='sum', alignToFrom=False): """ Summarize the data into interval buckets of a certain size. By default, the contents of each interval bucket are summed together. This is useful for counters where each increment represents a discrete event and retrieving a "per X" value requires summing all the events in that interval. Specifying 'avg' instead will return the mean for each bucket, which can be more useful when the value is a gauge that represents a certain value in time. 'max', 'min' or 'last' can also be specified. By default, buckets are caculated by rounding to the nearest interval. This works well for intervals smaller than a day. For example, 22:32 will end up in the bucket 22:00-23:00 when the interval=1hour. Passing alignToFrom=true will instead create buckets starting at the from time. In this case, the bucket for 22:32 depends on the from time. If from=6:30 then the 1hour bucket for 22:32 is 22:30-23:30. Example: .. code-block:: none &target=summarize(counter.errors, "1hour") # total errors per hour &target=summarize(nonNegativeDerivative(gauge.num_users), "1week") # new users per week &target=summarize(queue.size, "1hour", "avg") # average queue size per hour &target=summarize(queue.size, "1hour", "max") # maximum queue size during each hour &target=summarize(metric, "13week", "avg", true)&from=midnight+20100101 # 2010 Q1-4 """ results = [] delta = parseTimeOffset(intervalString) interval = delta.seconds + (delta.days * 86400) for series in seriesList: buckets = {} timestamps = range( int(series.start), int(series.end), int(series.step) ) datapoints = zip(timestamps, series) for (timestamp, value) in datapoints: if alignToFrom: bucketInterval = int((timestamp - series.start) / interval) else: bucketInterval = timestamp - (timestamp % interval) if bucketInterval not in buckets: buckets[bucketInterval] = [] if value is not None: buckets[bucketInterval].append(value) if alignToFrom: newStart = series.start newEnd = series.end else: newStart = series.start - (series.start % interval) newEnd = series.end - (series.end % interval) + interval newValues = [] for timestamp in range(newStart, newEnd, interval): if alignToFrom: newEnd = timestamp bucketInterval = int((timestamp - series.start) / interval) else: bucketInterval = timestamp - (timestamp % interval) bucket = buckets.get(bucketInterval, []) if bucket: if func == 'avg': newValues.append( float(sum(bucket)) / float(len(bucket)) ) elif func == 'last': newValues.append( bucket[len(bucket)-1] ) elif func == 'max': newValues.append( max(bucket) ) elif func == 'min': newValues.append( min(bucket) ) else: newValues.append( sum(bucket) ) else: newValues.append( None ) if alignToFrom: newEnd += interval newName = "summarize(%s, \"%s\")" % (series.name, intervalString) newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) return results
def test_parse_integer_raises_TypeError(self): with self.assertRaises(TypeError): time_ref = parseTimeOffset(1)
def test_parse_twelve_months_returns_360_days(self): time_ref = parseTimeOffset("12months") expected = timedelta(360) self.assertEquals(time_ref, expected)
def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError( self): with self.assertRaises(KeyError): time_ref = parseTimeOffset("Something")
def test_parse_two_years_returns_730_days(self): time_ref = parseTimeOffset("2years") expected = timedelta(730) self.assertEquals(time_ref, expected)
def test_parse_m_as_unit_raises_Exception(self): with self.assertRaises(Exception): time_ref = parseTimeOffset("1m")
def test_parse_ten_days(self): time_ref = parseTimeOffset("10days") expected = timedelta(10) self.assertEquals(time_ref, expected)
def test_parse_alpha_only_raises_KeyError(self): with self.assertRaises(KeyError): time_ref = parseTimeOffset("month")
def test_parse_plus_only_returns_zero(self): time_ref = parseTimeOffset("+") expected = timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_zero_days(self): time_ref = parseTimeOffset("0days") expected = timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_zero_days(self): time_ref = parseTimeOffset("0days") expected = datetime.timedelta(0) self.assertEquals(time_ref, expected)
def test_parse_minus_ten_days(self): time_ref = parseTimeOffset("-10days") expected = timedelta(-10) self.assertEquals(time_ref, expected)
def test_parse_five_seconds(self): time_ref = parseTimeOffset("5seconds") expected = datetime.timedelta(seconds=5) self.assertEquals(time_ref, expected)
def test_parse_five_seconds(self): time_ref = parseTimeOffset("5seconds") expected = timedelta(seconds=5) self.assertEquals(time_ref, expected)
def test_parse_five_hours(self): time_ref = parseTimeOffset("5hours") expected = datetime.timedelta(hours=5) self.assertEquals(time_ref, expected)
def test_parse_five_minutes(self): time_ref = parseTimeOffset("5minutes") expected = timedelta(minutes=5) self.assertEquals(time_ref, expected)
def test_parse_one_month_returns_thirty_days(self): time_ref = parseTimeOffset("1month") expected = datetime.timedelta(30) self.assertEquals(time_ref, expected)
def test_parse_five_hours(self): time_ref = parseTimeOffset("5hours") expected = timedelta(hours=5) self.assertEquals(time_ref, expected)
def test_parse_twelve_months_returns_360_days(self): time_ref = parseTimeOffset("12months") expected = datetime.timedelta(360) self.assertEquals(time_ref, expected)
def test_parse_five_weeks(self): time_ref = parseTimeOffset("5weeks") expected = timedelta(weeks=5) self.assertEquals(time_ref, expected)
def test_parse_two_years_returns_730_days(self): time_ref = parseTimeOffset("2years") expected = datetime.timedelta(730) self.assertEquals(time_ref, expected)
def test_parse_one_month_returns_thirty_days(self): time_ref = parseTimeOffset("1month") expected = timedelta(30) self.assertEquals(time_ref, expected)