def test_time_periods(self):
"""
Test if we can use time periods
"""
import pytz
utc = pytz.utc
# 2017-06-20 12:22:50
start = datetime(year=2017,
month=0o6,
day=20,
hour=12,
minute=22,
second=50)
# 2017-06-20 12:20:00
start_aligned = datetime(year=2017,
month=0o6,
day=20,
hour=12,
minute=20,
second=0)
interval = timedelta(minutes=5)
# 12:22:50+ 0:05:20 = 12:27:02
end = start + timedelta(minutes=5, seconds=22)
expected_periods = [
(
start_aligned.replace(tzinfo=utc),
start_aligned.replace(tzinfo=utc) + interval,
),
(
start_aligned.replace(tzinfo=utc) + interval,
start_aligned.replace(tzinfo=utc) + (2 * interval),
),
]
aligned = align_period_start(start, interval)
self.assertEqual(start_aligned.replace(tzinfo=utc),
aligned.replace(tzinfo=utc))
periods = list(generate_periods(start, interval, end))
self.assertEqual(expected_periods, periods)
pnow = datetime.utcnow().replace(tzinfo=pytz.utc)
start_for_one = pnow - interval
periods = list(generate_periods(start_for_one, interval, pnow))
self.assertEqual(len(periods), 1)
start_for_two = pnow - (2 * interval)
periods = list(generate_periods(start_for_two, interval, pnow))
self.assertEqual(len(periods), 2)
start_for_three = pnow - (3 * interval)
periods = list(generate_periods(start_for_three, interval, pnow))
self.assertEqual(len(periods), 3)
start_for_two_and_half = pnow - \
timedelta(seconds=(2.5 * interval.total_seconds()))
periods = list(generate_periods(start_for_two_and_half, interval,
pnow))
self.assertEqual(len(periods), 3)
def aggregate_past_periods(metric_data_q=None,
periods=None,
cleanup=True,
now=None,
max_since=None):
"""
Aggregate past metric data into longer periods
@param metric_data_q Query for metric data to use as input
(default: all MetricValues)
@param periods list of tuples (cutoff, aggregation) to be used
(default: settings.MONITORING_DATA_AGGREGATION)
@param cleanup flag if input data should be removed after aggregation
(default: True)
@param now arbitrary now moment to start calculation of cutoff
(default: current now)
@param max_since look for data no older than max_since
(default: 1 year)
"""
utc = pytz.utc
if now is None:
now = datetime.utcnow().replace(tzinfo=utc)
if metric_data_q is None:
metric_data_q = MetricValue.objects.all()
if periods is None:
periods = settings.MONITORING_DATA_AGGREGATION
max_since = max_since or now - timedelta(days=356)
previous_cutoff = None
counter = 0
now = adjust_now_to_noon(now)
# start from the end, oldest one first
for cutoff_base, aggregation_period in reversed(periods):
since = previous_cutoff or max_since
until = now - cutoff_base
if since > until:
log.debug(
"Wrong period boundaries, end %s is before start %s, agg: %s",
until, since, aggregation_period)
previous_cutoff = max(until, since)
continue
log.debug(
"aggregation params: cutoff: %s agg period: %s"
"\n since: '%s' until '%s', but previous cutoff:"
" '%s', aggregate to '%s'", cutoff_base, aggregation_period, since,
until, previous_cutoff, aggregation_period)
periods = generate_periods(since, aggregation_period, end=until)
# for each target period we select mertic values within it
# and extract service, resource, event type and label combinations
# then, for each distinctive set, calculate per-metric aggregate values
for period_start, period_end in periods:
log.debug('period %s - %s (%s s)', period_start, period_end,
period_end - period_start)
ret = aggregate_period(period_start, period_end, metric_data_q,
cleanup)
counter += ret
previous_cutoff = until
return counter
def get_metrics_for(self, metric_name,
valid_from=None,
valid_to=None,
interval=None,
service=None,
label=None,
user=None,
resource=None,
event_type=None,
service_type=None,
group_by=None,
resource_type=None):
"""
Returns metric data for given metric. Returned dataset contains list of periods and values in that periods
"""
utc = pytz.utc
default_interval = False
now = datetime.utcnow().replace(tzinfo=utc)
if not interval:
default_interval = True
interval = timedelta(seconds=60)
if not isinstance(interval, timedelta):
interval = timedelta(seconds=interval)
valid_from = valid_from or (now - interval)
valid_to = valid_to or now
if (not interval or default_interval) and (
valid_to - valid_from).total_seconds() > 24 * 3600:
default_interval = True
interval = timedelta(seconds=3600)
if not isinstance(interval, timedelta):
interval = timedelta(seconds=interval)
metric = Metric.objects.get(name=metric_name)
out = {'metric': metric.name,
'input_valid_from': valid_from.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'input_valid_to': valid_to.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'interval': interval.total_seconds(),
'label': label.name if label else None,
'type': metric.type,
'axis_label': metric.unit,
'data': []}
periods = generate_periods(valid_from, interval, valid_to, align=False)
for pstart, pend in periods:
pdata = self.get_metrics_data(metric_name, pstart, pend,
interval=interval,
service=service,
label=label,
user=user,
event_type=event_type,
service_type=service_type,
resource=resource,
resource_type=resource_type,
group_by=group_by)
out['data'].append({
'valid_from': pstart.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'valid_to': pend.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': pdata
})
return out
def process_requests(self, service, requests, valid_from, valid_to):
"""
Processes request list for specific service, generate stats
"""
interval = service.check_interval
periods = generate_periods(valid_from, interval, valid_to)
for pstart, pend in periods:
requests_batch = requests.filter(
created__gte=pstart, created__lt=pend)
self.process_requests_batch(service, requests_batch, pstart, pend)
def test_time_periods(self):
"""
Test if we can use time periods
"""
import pytz
utc = pytz.utc
# 2017-06-20 12:22:50
start = datetime(
year=2017,
month=0o6,
day=20,
hour=12,
minute=22,
second=50)
# 2017-06-20 12:20:00
start_aligned = datetime(
year=2017,
month=0o6,
day=20,
hour=12,
minute=20,
second=0)
interval = timedelta(minutes=5)
# 12:22:50+ 0:05:20 = 12:27:02
end = start + timedelta(minutes=5, seconds=22)
expected_periods = [(start_aligned.replace(tzinfo=utc),
start_aligned.replace(tzinfo=utc) + interval,),
(start_aligned.replace(tzinfo=utc) + interval,
start_aligned.replace(
tzinfo=utc) + (2 * interval),),
]
aligned = align_period_start(start, interval)
self.assertEqual(
start_aligned.replace(tzinfo=utc),
aligned.replace(tzinfo=utc))
periods = list(generate_periods(start, interval, end))
self.assertEqual(expected_periods, periods)
pnow = datetime.utcnow().replace(tzinfo=pytz.utc)
start_for_one = pnow - interval
periods = list(generate_periods(start_for_one, interval, pnow))
self.assertEqual(len(periods), 1)
start_for_two = pnow - (2 * interval)
periods = list(generate_periods(start_for_two, interval, pnow))
self.assertEqual(len(periods), 2)
start_for_three = pnow - (3 * interval)
periods = list(generate_periods(start_for_three, interval, pnow))
self.assertEqual(len(periods), 3)
start_for_two_and_half = pnow - \
timedelta(seconds=(2.5 * interval.total_seconds()))
periods = list(
generate_periods(
start_for_two_and_half,
interval,
pnow))
self.assertEqual(len(periods), 3)
