def test_time_periods(self): """ Test if we can use time periods """ import pytz utc = pytz.utc # 2017-06-20 12:22:50 start = datetime(year=2017, month=0o6, day=20, hour=12, minute=22, second=50) # 2017-06-20 12:20:00 start_aligned = datetime(year=2017, month=0o6, day=20, hour=12, minute=20, second=0) interval = timedelta(minutes=5) # 12:22:50+ 0:05:20 = 12:27:02 end = start + timedelta(minutes=5, seconds=22) expected_periods = [ ( start_aligned.replace(tzinfo=utc), start_aligned.replace(tzinfo=utc) + interval, ), ( start_aligned.replace(tzinfo=utc) + interval, start_aligned.replace(tzinfo=utc) + (2 * interval), ), ] aligned = align_period_start(start, interval) self.assertEqual(start_aligned.replace(tzinfo=utc), aligned.replace(tzinfo=utc)) periods = list(generate_periods(start, interval, end)) self.assertEqual(expected_periods, periods) pnow = datetime.utcnow().replace(tzinfo=pytz.utc) start_for_one = pnow - interval periods = list(generate_periods(start_for_one, interval, pnow)) self.assertEqual(len(periods), 1) start_for_two = pnow - (2 * interval) periods = list(generate_periods(start_for_two, interval, pnow)) self.assertEqual(len(periods), 2) start_for_three = pnow - (3 * interval) periods = list(generate_periods(start_for_three, interval, pnow)) self.assertEqual(len(periods), 3) start_for_two_and_half = pnow - \ timedelta(seconds=(2.5 * interval.total_seconds())) periods = list(generate_periods(start_for_two_and_half, interval, pnow)) self.assertEqual(len(periods), 3)
def aggregate_past_periods(metric_data_q=None, periods=None, cleanup=True, now=None, max_since=None): """ Aggregate past metric data into longer periods @param metric_data_q Query for metric data to use as input (default: all MetricValues) @param periods list of tuples (cutoff, aggregation) to be used (default: settings.MONITORING_DATA_AGGREGATION) @param cleanup flag if input data should be removed after aggregation (default: True) @param now arbitrary now moment to start calculation of cutoff (default: current now) @param max_since look for data no older than max_since (default: 1 year) """ utc = pytz.utc if now is None: now = datetime.utcnow().replace(tzinfo=utc) if metric_data_q is None: metric_data_q = MetricValue.objects.all() if periods is None: periods = settings.MONITORING_DATA_AGGREGATION max_since = max_since or now - timedelta(days=356) previous_cutoff = None counter = 0 now = adjust_now_to_noon(now) # start from the end, oldest one first for cutoff_base, aggregation_period in reversed(periods): since = previous_cutoff or max_since until = now - cutoff_base if since > until: log.debug( "Wrong period boundaries, end %s is before start %s, agg: %s", until, since, aggregation_period) previous_cutoff = max(until, since) continue log.debug( "aggregation params: cutoff: %s agg period: %s" "\n since: '%s' until '%s', but previous cutoff:" " '%s', aggregate to '%s'", cutoff_base, aggregation_period, since, until, previous_cutoff, aggregation_period) periods = generate_periods(since, aggregation_period, end=until) # for each target period we select mertic values within it # and extract service, resource, event type and label combinations # then, for each distinctive set, calculate per-metric aggregate values for period_start, period_end in periods: log.debug('period %s - %s (%s s)', period_start, period_end, period_end - period_start) ret = aggregate_period(period_start, period_end, metric_data_q, cleanup) counter += ret previous_cutoff = until return counter
def get_metrics_for(self, metric_name, valid_from=None, valid_to=None, interval=None, service=None, label=None, user=None, resource=None, event_type=None, service_type=None, group_by=None, resource_type=None): """ Returns metric data for given metric. Returned dataset contains list of periods and values in that periods """ utc = pytz.utc default_interval = False now = datetime.utcnow().replace(tzinfo=utc) if not interval: default_interval = True interval = timedelta(seconds=60) if not isinstance(interval, timedelta): interval = timedelta(seconds=interval) valid_from = valid_from or (now - interval) valid_to = valid_to or now if (not interval or default_interval) and ( valid_to - valid_from).total_seconds() > 24 * 3600: default_interval = True interval = timedelta(seconds=3600) if not isinstance(interval, timedelta): interval = timedelta(seconds=interval) metric = Metric.objects.get(name=metric_name) out = {'metric': metric.name, 'input_valid_from': valid_from.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 'input_valid_to': valid_to.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 'interval': interval.total_seconds(), 'label': label.name if label else None, 'type': metric.type, 'axis_label': metric.unit, 'data': []} periods = generate_periods(valid_from, interval, valid_to, align=False) for pstart, pend in periods: pdata = self.get_metrics_data(metric_name, pstart, pend, interval=interval, service=service, label=label, user=user, event_type=event_type, service_type=service_type, resource=resource, resource_type=resource_type, group_by=group_by) out['data'].append({ 'valid_from': pstart.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 'valid_to': pend.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 'data': pdata }) return out
def process_requests(self, service, requests, valid_from, valid_to): """ Processes request list for specific service, generate stats """ interval = service.check_interval periods = generate_periods(valid_from, interval, valid_to) for pstart, pend in periods: requests_batch = requests.filter( created__gte=pstart, created__lt=pend) self.process_requests_batch(service, requests_batch, pstart, pend)
def test_time_periods(self): """ Test if we can use time periods """ import pytz utc = pytz.utc # 2017-06-20 12:22:50 start = datetime( year=2017, month=0o6, day=20, hour=12, minute=22, second=50) # 2017-06-20 12:20:00 start_aligned = datetime( year=2017, month=0o6, day=20, hour=12, minute=20, second=0) interval = timedelta(minutes=5) # 12:22:50+ 0:05:20 = 12:27:02 end = start + timedelta(minutes=5, seconds=22) expected_periods = [(start_aligned.replace(tzinfo=utc), start_aligned.replace(tzinfo=utc) + interval,), (start_aligned.replace(tzinfo=utc) + interval, start_aligned.replace( tzinfo=utc) + (2 * interval),), ] aligned = align_period_start(start, interval) self.assertEqual( start_aligned.replace(tzinfo=utc), aligned.replace(tzinfo=utc)) periods = list(generate_periods(start, interval, end)) self.assertEqual(expected_periods, periods) pnow = datetime.utcnow().replace(tzinfo=pytz.utc) start_for_one = pnow - interval periods = list(generate_periods(start_for_one, interval, pnow)) self.assertEqual(len(periods), 1) start_for_two = pnow - (2 * interval) periods = list(generate_periods(start_for_two, interval, pnow)) self.assertEqual(len(periods), 2) start_for_three = pnow - (3 * interval) periods = list(generate_periods(start_for_three, interval, pnow)) self.assertEqual(len(periods), 3) start_for_two_and_half = pnow - \ timedelta(seconds=(2.5 * interval.total_seconds())) periods = list( generate_periods( start_for_two_and_half, interval, pnow)) self.assertEqual(len(periods), 3)