def get(self):
"""Presents nice recent statistics.
It fetches data from the 'JSON' API.
"""
# Preloads the data to save a complete request.
resolution = self.request.params.get('resolution', 'hours')
if resolution not in ('days', 'hours', 'minutes'):
resolution = 'hours'
duration = utils.get_request_as_int(self.request, 'duration', 120, 1,
1000)
description = _GVIZ_DESCRIPTION.copy()
description.update(
stats_framework_gviz.get_description_key(resolution))
table = stats_framework.get_stats(stats.STATS_HANDLER, resolution,
None, duration, True)
params = {
'duration':
duration,
'initial_data':
gviz_api.DataTable(
description, table).ToJSon(columns_order=_GVIZ_COLUMNS_ORDER),
'now':
datetime.datetime.utcnow(),
'resolution':
resolution,
}
self.response.write(template.render('isolate/stats.html', params))
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None,
None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes',
self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def get_json(request, response, handler, resolution, description, order):
"""Returns the statistic data as a Google Visualization compatible reply.
The return can be either JSON or JSONP, depending if the header
'X-DataSource-Auth' is set in the request.
Note that this is not real JSON, as explained in
developers.google.com/chart/interactive/docs/dev/implementing_data_source
Exposes the data in the format described at
https://developers.google.com/chart/interactive/docs/reference#dataparam
and
https://developers.google.com/chart/interactive/docs/querylanguage
Arguments:
- request: A webapp2.Request.
- response: A webapp2.Response.
- handler: A StatisticsFramework.
- resolution: One of 'days', 'hours' or 'minutes'.
- description: Dict describing the columns.
- order: List describing the order to use for the columns.
Raises:
ValueError if a 400 should be returned.
"""
tqx_args = process_tqx(request.params.get('tqx', ''))
duration = utils.get_request_as_int(request, 'duration', 120, 1, 256)
now = None
now_text = request.params.get('now')
if now_text:
now = utils.parse_datetime(now_text)
table = stats_framework.get_stats(handler, resolution, now, duration, True)
return get_json_raw(request, response, table, description, order, tqx_args)
def send_response(self, res_type_info):
"""Presents nice recent statistics.
It preloads data in the template for maximum responsiveness and
interactively fetches data from the JSON API.
"""
# Preloads the data to save a complete request.
resolution = self.request.params.get('resolution', 'hours')
if resolution not in stats_framework.RESOLUTIONS:
self.abort(404)
duration = utils.get_request_as_int(self.request,
'duration',
default=120,
min_value=1,
max_value=1000)
now = utils.get_request_as_datetime(self.request, 'now')
now = now or datetime.datetime.utcnow()
description = res_type_info.DESCRIPTION.copy()
description.update(
stats_framework_gviz.get_description_key(resolution))
stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution,
now, duration, False)
template_data = self.process_data(description, stats_data)
template_data['duration'] = duration
template_data['now'] = now
template_data['resolution'] = resolution
self.response.write(
template.render(res_type_info.TEMPLATE, template_data))
def Stats(self, request, context):
res = self._RESOLUTION_MAP.get(request.resolution)
if not res:
context.set_code(StatusCode.INVALID_ARGUMENT)
context.set_details('Invalid resolution')
return None
if not 1 <= request.page_size <= 1000:
context.set_code(StatusCode.INVALID_ARGUMENT)
context.set_details(
'Invalid page_size; must be between 1 and 1000')
return None
if request.latest_time.seconds:
now = request.latest_time.ToDatetime()
else:
now = utils.utcnow()
# Round time to the minute.
now = datetime.datetime(*now.timetuple()[:5], tzinfo=now.tzinfo)
entities = stats_framework.get_stats(stats.STATS_HANDLER, res, now,
request.page_size, False)
out = isolated_pb2.StatsResponse()
for s in entities:
stats.snapshot_to_proto(s, out.measurements.add())
logging.info('Found %d entities', len(entities))
return out
def get_json(request, response, handler, resolution, description, order):
"""Returns the statistic data as a Google Visualization compatible reply.
The return can be either JSON or JSONP, depending if the header
'X-DataSource-Auth' is set in the request.
Note that this is not real JSON, as explained in
developers.google.com/chart/interactive/docs/dev/implementing_data_source
Exposes the data in the format described at
https://developers.google.com/chart/interactive/docs/reference#dataparam
and
https://developers.google.com/chart/interactive/docs/querylanguage
Arguments:
- request: A webapp2.Request.
- response: A webapp2.Response.
- handler: A StatisticsFramework.
- resolution: One of 'days', 'hours' or 'minutes'.
- description: Dict describing the columns.
- order: List describing the order to use for the columns.
Raises:
ValueError if a 400 should be returned.
"""
tqx_args = process_tqx(request.params.get('tqx', ''))
duration = utils.get_request_as_int(request, 'duration', 120, 1, 256)
now = utils.get_request_as_datetime(request, 'now')
table = stats_framework.get_stats(handler, resolution, now, duration, True)
return get_json_raw(request, response, table, description, order, tqx_args)
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None, None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(
stats.STATS_HANDLER, 'minutes', self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def get(self2):
self2.response.headers['Content-Type'] = (
'application/json; charset=utf-8')
duration = int(self2.request.get('duration', 120))
now = self2.request.get('now')
resolution = self2.request.get('resolution')
data = stats_framework.get_stats(
self.h, resolution, now, duration, True)
self2.response.write(stats_framework.utils.encode_to_json(data))
def get(self2):
self2.response.headers['Content-Type'] = (
'application/json; charset=utf-8')
duration = int(self2.request.get('duration', 120))
now = self2.request.get('now')
resolution = self2.request.get('resolution')
data = stats_framework.get_stats(
self.h, resolution, now, duration, True)
self2.response.write(stats_framework.utils.encode_to_json(data))
def send_response(self, res_type_info, resolution):
if resolution not in stats_framework.RESOLUTIONS:
self.abort(404)
duration = utils.get_request_as_int(self.request, "duration", default=120, min_value=1, max_value=1000)
now = utils.get_request_as_datetime(self.request, "now")
description = res_type_info.DESCRIPTION.copy()
description.update(stats_framework_gviz.get_description_key(resolution))
stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False)
tqx_args = tqx_args = stats_framework_gviz.process_tqx(self.request.params.get("tqx", ""))
try:
stats_framework_gviz.get_json_raw(
self.request, self.response, self.get_table(stats_data), description, res_type_info.ORDER, tqx_args
)
except ValueError as e:
self.abort(400, str(e))
def get(self):
"""Presents nice recent statistics.
It fetches data from the 'JSON' API.
"""
# Preloads the data to save a complete request.
resolution = self.request.params.get('resolution', 'hours')
duration = utils.get_request_as_int(self.request, 'duration', 120, 1, 1000)
description = _GVIZ_DESCRIPTION.copy()
description.update(stats_framework_gviz.get_description_key(resolution))
table = stats_framework.get_stats(
stats.STATS_HANDLER, resolution, None, duration, True)
params = {
'duration': duration,
'initial_data': gviz_api.DataTable(description, table).ToJSon(
columns_order=_GVIZ_COLUMNS_ORDER),
'now': datetime.datetime.utcnow(),
'resolution': resolution,
}
self.response.write(template.render('isolate/stats.html', params))
def send_response(self, res_type_info):
"""Presents nice recent statistics.
It preloads data in the template for maximum responsiveness and
interactively fetches data from the JSON API.
"""
# Preloads the data to save a complete request.
resolution = self.request.params.get("resolution", "hours")
if resolution not in stats_framework.RESOLUTIONS:
self.abort(404)
duration = utils.get_request_as_int(self.request, "duration", default=120, min_value=1, max_value=1000)
now = utils.get_request_as_datetime(self.request, "now")
now = now or datetime.datetime.utcnow()
description = res_type_info.DESCRIPTION.copy()
description.update(stats_framework_gviz.get_description_key(resolution))
stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False)
template_data = self.process_data(description, stats_data)
template_data["duration"] = duration
template_data["now"] = now
template_data["resolution"] = resolution
self.response.write(template.render(res_type_info.TEMPLATE, template_data))
def send_response(self, res_type_info, resolution):
if resolution not in stats_framework.RESOLUTIONS:
self.abort(404)
duration = utils.get_request_as_int(self.request,
'duration',
default=120,
min_value=1,
max_value=1000)
now = utils.get_request_as_datetime(self.request, 'now')
description = res_type_info.DESCRIPTION.copy()
description.update(
stats_framework_gviz.get_description_key(resolution))
stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution,
now, duration, False)
tqx_args = tqx_args = stats_framework_gviz.process_tqx(
self.request.params.get('tqx', ''))
try:
stats_framework_gviz.get_json_raw(self.request, self.response,
self.get_table(stats_data),
description, res_type_info.ORDER,
tqx_args)
except ValueError as e:
self.abort(400, str(e))
def test_framework_last_few(self):
called = []
def gen_data(start, end):
"""Returns fake statistics."""
self.assertEqual(start + 60, end)
called.append(start)
return Snapshot(requests=1,
b=1,
inner=InnerSnapshot(c='%d,' % len(called)))
handler = stats_framework.StatisticsFramework('test_framework',
Snapshot, gen_data)
now = get_now()
self.mock_now(now, 0)
handler._set_last_processed_time(
strip_seconds(now) - datetime.timedelta(seconds=3 * 60))
i = handler.process_next_chunk(1)
self.assertEqual(2, i)
self.assertEqual(1, stats_framework.StatsRoot.query().count())
self.assertEqual(1, handler.stats_day_cls.query().count())
self.assertEqual(1, handler.stats_hour_cls.query().count())
self.assertEqual(2, handler.stats_minute_cls.query().count())
root = handler.root_key.get()
self.assertEqual(
strip_seconds(now) - datetime.timedelta(seconds=60),
root.timestamp)
# Trying to process more won't do anything.
i = handler.process_next_chunk(1)
self.assertEqual(0, i)
root = handler.root_key.get()
self.assertEqual(
strip_seconds(now) - datetime.timedelta(seconds=60),
root.timestamp)
expected = [
{
'key': '2010-01-02',
'requests': 0,
'b': 0.0,
'd': [],
'inner': {
'c': u''
},
},
]
self.assertEqual(
expected, stats_framework.get_stats(handler, 'days', now, 100,
True))
expected = [
{
'key': '2010-01-02T03',
'requests': 2,
'b': 2.0,
'd': [],
'inner': {
'c': u'1,2,'
},
},
]
self.assertEqual(
expected,
stats_framework.get_stats(handler, 'hours', now, 100, True))
expected = [
{
'key': '2010-01-02T03:03',
'requests': 1,
'b': 1.0,
'd': [],
'inner': {
'c': u'2,'
},
},
{
'key': '2010-01-02T03:02',
'requests': 1,
'b': 1.0,
'd': [],
'inner': {
'c': u'1,'
},
},
]
self.assertEqual(
expected,
stats_framework.get_stats(handler, 'minutes', now, 100, True))
def test_framework_last_few(self):
called = []
def gen_data(start, end):
"""Returns fake statistics."""
self.assertEqual(start + 60, end)
called.append(start)
return Snapshot(
requests=1, b=1, inner=InnerSnapshot(c='%d,' % len(called)))
handler = stats_framework.StatisticsFramework(
'test_framework', Snapshot, gen_data)
now = get_now()
self.mock_now(now, 0)
handler._set_last_processed_time(
strip_seconds(now) - datetime.timedelta(seconds=3*60))
i = handler.process_next_chunk(1)
self.assertEqual(2, i)
self.assertEqual(1, stats_framework.StatsRoot.query().count())
self.assertEqual(1, handler.stats_day_cls.query().count())
self.assertEqual(1, handler.stats_hour_cls.query().count())
self.assertEqual(2, handler.stats_minute_cls.query().count())
root = handler.root_key.get()
self.assertEqual(
strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp)
# Trying to process more won't do anything.
i = handler.process_next_chunk(1)
self.assertEqual(0, i)
root = handler.root_key.get()
self.assertEqual(
strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp)
expected = [
{
'key': now.date(),
'requests': 0,
'b': 0.0,
'd': [],
'inner': {'c': u''},
},
]
self.assertEqual(
expected, stats_framework.get_stats(handler, 'days', now, 100, True))
expected = [
{
'key': datetime.datetime(*now.timetuple()[:4]),
'requests': 2,
'b': 2.0,
'd': [],
'inner': {'c': u'1,2,'},
},
]
self.assertEqual(
expected, stats_framework.get_stats(handler, 'hours', now, 100, True))
expected = [
{
'key': datetime.datetime(
*(now - datetime.timedelta(seconds=60)).timetuple()[:5]),
'requests': 1,
'b': 1.0,
'd': [],
'inner': {'c': u'2,'},
},
{
'key': datetime.datetime(
*(now - datetime.timedelta(seconds=120)).timetuple()[:5]),
'requests': 1,
'b': 1.0,
'd': [],
'inner': {'c': u'1,'},
},
]
self.assertEqual(
expected, stats_framework.get_stats(handler, 'minutes', now, 100, True))