def incr(self, project, name, timestamp, value=1, filters={}, callback=None, **kwargs):
"""Add value to metric counter
:param project: project name
:param name: metric name
:param timestamp: timestamp name
:param value: increment value
:param filters: dict of filters
:param \*\*kwargs: additional kwargs
"""
timestamp = timestamp_to_datetime(timestamp)
if filters:
filters = dict(filters)
for period in self._application.config['PERIODS']:
if filters:
for fname, fvalue in filters.items():
self.save_value(project, name, period,
get_by_period(date_min(timestamp, period), period), fname, fvalue, value)
self.save_value(project, name, period,
get_by_period(date_min(timestamp, period), period), None, None, value)
self.save_metric_meta(project, name, filters)
self.update_stats('incr')
if callback:
callback(True)
def get_range(self, data, from_date, to_date, period,
filter_name=None, filter_value=None):
"""Shortcut for datarange
:param data: data list
:param from_date:
:param to_date:
:param period:
"""
from_date = date_min(from_date, period)
to_date = date_max(to_date, period)
data = self.filter_data_by_period(data, from_date, to_date)
groups = groupby(data, key=lambda x: get_by_period(date_min(x['timestamp'], period), period))
for group, group_items in groups:
if not group_items:
yield (group, 0)
continue
group_sum = 0
if not filter_name:
group_sum = sum([x['value'] for x in group_items])
else:
group_sum = sum([x['value'] for x in group_items
if (x['filter_name'] == filter_name)
and (x['filter_value'] == filter_value)])
yield (group, group_sum)
def get_range(self,
data,
from_date,
to_date,
period,
filter_name=None,
filter_value=None):
"""Shortcut for datarange
:param data: data list
:param from_date:
:param to_date:
:param period:
"""
from_date = date_min(from_date, period)
to_date = date_max(to_date, period)
data = self.filter_data_by_period(data, from_date, to_date)
groups = groupby(data,
key=lambda x: get_by_period(
date_min(x['timestamp'], period), period))
for group, group_items in groups:
if not group_items:
yield (group, 0)
continue
group_sum = 0
if not filter_name:
group_sum = sum([x['value'] for x in group_items])
else:
group_sum = sum([
x['value'] for x in group_items
if (x['filter_name'] == filter_name) and (
x['filter_value'] == filter_value)
])
yield (group, group_sum)
def convert_date_range(self, from_date, to_date, period):
"""Convert str from_date and to_data objects to
datetime object
:param from_date: from date string
:param to_date: to date string
:return: tuple (from_date, to_date)
"""
from_date = timestamp_to_datetime(from_date, DATE_FILTER_FORMAT) if from_date else from_date
to_date = timestamp_to_datetime(to_date, DATE_FILTER_FORMAT) if to_date else to_date
return date_min(from_date, period), date_max(to_date, period)
def incr(self, project, name, timestamp, value=1, filters={}, callback=None, **kwargs):
"""Make incr in redis hash
:param project: project name
:param name: metric name
:param timestamp: timstamp object
:param value: increment value
:param filters: dict of filters
"""
pipe = self.client.pipeline(transactional=True)
pipe.select(self.selected_db)
timestamp = timestamp_to_datetime(timestamp)
if filters:
filters = dict(filters)
for period in self._application.config['PERIODS']:
if filters:
for fname, fvalue in filters.iteritems():
if not isinstance(fvalue, (list, tuple)):
# Make incr for all values
fvalue = [fvalue]
for fvalue_item in fvalue:
pipe.hincrby(self.make_key(
project, name, period, {fname: fvalue_item}),
get_by_period(date_min(timestamp, period), period), value)
pipe.hincrby(self.make_key(project, name, period), get_by_period(date_min(timestamp, period), period), value)
self.save_metric_meta(pipe, project, name, filters)
res = (yield gen.Task(pipe.execute))
self.update_stats('incr')
if callback:
callback(res)
def incr(self,
project,
name,
timestamp,
value=1,
filters={},
callback=None,
**kwargs):
"""Add value to metric counter
:param project: project name
:param name: metric name
:param timestamp: timestamp name
:param value: increment value
:param filters: dict of filters
:param \*\*kwargs: additional kwargs
"""
timestamp = timestamp_to_datetime(timestamp)
if filters:
filters = dict(filters)
for period in self._application.config['PERIODS']:
if filters:
for fname, fvalue in list(filters.items()):
self.save_value(
project, name, period,
get_by_period(date_min(timestamp, period), period),
fname, fvalue, value)
self.save_value(project, name, period,
get_by_period(date_min(timestamp, period), period),
None, None, value)
self.save_metric_meta(project, name, filters)
self.update_stats('incr')
if callback:
callback(True)
def convert_date_range(self, from_date, to_date, period):
"""Convert str from_date and to_data objects to
datetime object
:param from_date: from date string
:param to_date: to date string
:return: tuple (from_date, to_date)
"""
from_date = timestamp_to_datetime(
from_date, DATE_FILTER_FORMAT) if from_date else from_date
to_date = timestamp_to_datetime(
to_date, DATE_FILTER_FORMAT) if to_date else to_date
return date_min(from_date, period), date_max(to_date, period)
def filter_by_period(self, data, period, from_date=None, to_date=None):
"""Fulter statistics by `from_date` and `to_date`
:param from_data:
:param to_date:
:return: ifilter generator
"""
from_date = date_min(from_date, period)
to_date = date_max(to_date, period)
# Convert datestring to datetime object in list
from_date = get_by_period(from_date, period)
to_date = get_by_period(to_date, period)
return sorted(ifilter(lambda x: (True if from_date is None else int(x[0]) >= from_date) and \
(True if to_date is None else int(x[0]) <= to_date),
imap(lambda item: (int(item[0]), int(item[1])), data)),
key=lambda x: x[0])
def filter_by_period(self, data, period, from_date=None, to_date=None):
"""Fulter statistics by `from_date` and `to_date`
:param from_data:
:param to_date:
:return: ifilter generator
"""
from_date = date_min(from_date, period)
to_date = date_max(to_date, period)
# Convert datestring to datetime object in list
from_date = get_by_period(from_date, period)
to_date = get_by_period(to_date, period)
return sorted(filter(lambda x: (True if from_date is None else int(x[0]) >= from_date) and \
(True if to_date is None else int(x[0]) <= to_date),
map(lambda item: (int(item[0]), int(item[1])), data)),
key=lambda x: x[0])
def test_date_min_and_max(self):
d = datetime(2012, 11, 1, 4, 5, 2)
self.assertEquals(date_min(d, "year"), datetime(2012, 1, 1, 0, 0))
self.assertEquals(date_min(d, "month"), datetime(2012, 11, 1, 0, 0))
self.assertEquals(date_min(d, "week"), datetime(2012, 10, 29, 0, 0))
self.assertEquals(date_min(d, "day"), datetime(2012, 11, 1, 0, 0))
self.assertEquals(date_min(d, "hour"), datetime(2012, 11, 1, 4, 0))
self.assertEquals(date_min(d, "minute"), datetime(2012, 11, 1, 4, 5))
self.assertEquals(date_max(d, "year"), datetime(2012, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "month"), datetime(2012, 11, 30, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "week"), datetime(2012, 11, 4, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "day"), datetime(2012, 11, 1, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "hour"), datetime(2012, 11, 1, 4, 59, 59, 999999))
self.assertEquals(date_max(d, "minute"), datetime(2012, 11, 1, 4, 5, 59, 999999))
d = datetime(2013, 12, 30, 4, 5, 2)
self.assertEquals(date_min(d, "year"), datetime(2013, 1, 1, 0, 0, 0))
self.assertEquals(date_min(d, "month"), datetime(2013, 12, 1, 0, 0, 0))
self.assertEquals(date_min(d, "week"), datetime(2013, 12, 30, 0, 0, 0))
self.assertEquals(date_min(d, "day"), datetime(2013, 12, 30, 0, 0, 0, 0))
self.assertEquals(date_min(d, "hour"), datetime(2013, 12, 30, 4, 0, 0))
self.assertEquals(date_min(d, "minute"), datetime(2013, 12, 30, 4, 5, 0))
self.assertEquals(date_max(d, "year"), datetime(2013, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "month"), datetime(2013, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "week"), datetime(2014, 1, 5, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "day"), datetime(2013, 12, 30, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "hour"), datetime(2013, 12, 30, 4, 59, 59, 999999))
self.assertEquals(date_max(d, "minute"), datetime(2013, 12, 30, 4, 5, 59, 999999))
def tmin(d, period):
return get_by_period(date_min(d, period), period)
def test_date_min_and_max(self):
d = datetime(2012, 11, 1, 4, 5, 2)
self.assertEquals(date_min(d, "year"), datetime(2012, 1, 1, 0, 0))
self.assertEquals(date_min(d, "month"), datetime(2012, 11, 1, 0, 0))
self.assertEquals(date_min(d, "week"), datetime(2012, 10, 29, 0, 0))
self.assertEquals(date_min(d, "day"), datetime(2012, 11, 1, 0, 0))
self.assertEquals(date_min(d, "hour"), datetime(2012, 11, 1, 4, 0))
self.assertEquals(date_min(d, "minute"), datetime(2012, 11, 1, 4, 5))
self.assertEquals(date_max(d, "year"),
datetime(2012, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "month"),
datetime(2012, 11, 30, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "week"),
datetime(2012, 11, 4, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "day"),
datetime(2012, 11, 1, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "hour"),
datetime(2012, 11, 1, 4, 59, 59, 999999))
self.assertEquals(date_max(d, "minute"),
datetime(2012, 11, 1, 4, 5, 59, 999999))
d = datetime(2013, 12, 30, 4, 5, 2)
self.assertEquals(date_min(d, "year"), datetime(2013, 1, 1, 0, 0, 0))
self.assertEquals(date_min(d, "month"), datetime(2013, 12, 1, 0, 0, 0))
self.assertEquals(date_min(d, "week"), datetime(2013, 12, 30, 0, 0, 0))
self.assertEquals(date_min(d, "day"), datetime(2013, 12, 30, 0, 0, 0,
0))
self.assertEquals(date_min(d, "hour"), datetime(2013, 12, 30, 4, 0, 0))
self.assertEquals(date_min(d, "minute"),
datetime(2013, 12, 30, 4, 5, 0))
self.assertEquals(date_max(d, "year"),
datetime(2013, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "month"),
datetime(2013, 12, 31, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "week"),
datetime(2014, 1, 5, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "day"),
datetime(2013, 12, 30, 23, 59, 59, 999999))
self.assertEquals(date_max(d, "hour"),
datetime(2013, 12, 30, 4, 59, 59, 999999))
self.assertEquals(date_max(d, "minute"),
datetime(2013, 12, 30, 4, 5, 59, 999999))
def tmin(d, period):
return get_by_period(date_min(d, period), period)
