def get(self): """Presents nice recent statistics. It fetches data from the 'JSON' API. """ # Preloads the data to save a complete request. resolution = self.request.params.get('resolution', 'hours') if resolution not in ('days', 'hours', 'minutes'): resolution = 'hours' duration = utils.get_request_as_int(self.request, 'duration', 120, 1, 1000) description = _GVIZ_DESCRIPTION.copy() description.update( stats_framework_gviz.get_description_key(resolution)) table = stats_framework.get_stats(stats.STATS_HANDLER, resolution, None, duration, True) params = { 'duration': duration, 'initial_data': gviz_api.DataTable( description, table).ToJSon(columns_order=_GVIZ_COLUMNS_ORDER), 'now': datetime.datetime.utcnow(), 'resolution': resolution, } self.response.write(template.render('isolate/stats.html', params))
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def get_json(request, response, handler, resolution, description, order): """Returns the statistic data as a Google Visualization compatible reply. The return can be either JSON or JSONP, depending if the header 'X-DataSource-Auth' is set in the request. Note that this is not real JSON, as explained in developers.google.com/chart/interactive/docs/dev/implementing_data_source Exposes the data in the format described at https://developers.google.com/chart/interactive/docs/reference#dataparam and https://developers.google.com/chart/interactive/docs/querylanguage Arguments: - request: A webapp2.Request. - response: A webapp2.Response. - handler: A StatisticsFramework. - resolution: One of 'days', 'hours' or 'minutes'. - description: Dict describing the columns. - order: List describing the order to use for the columns. Raises: ValueError if a 400 should be returned. """ tqx_args = process_tqx(request.params.get('tqx', '')) duration = utils.get_request_as_int(request, 'duration', 120, 1, 256) now = None now_text = request.params.get('now') if now_text: now = utils.parse_datetime(now_text) table = stats_framework.get_stats(handler, resolution, now, duration, True) return get_json_raw(request, response, table, description, order, tqx_args)
def send_response(self, res_type_info): """Presents nice recent statistics. It preloads data in the template for maximum responsiveness and interactively fetches data from the JSON API. """ # Preloads the data to save a complete request. resolution = self.request.params.get('resolution', 'hours') if resolution not in stats_framework.RESOLUTIONS: self.abort(404) duration = utils.get_request_as_int(self.request, 'duration', default=120, min_value=1, max_value=1000) now = utils.get_request_as_datetime(self.request, 'now') now = now or datetime.datetime.utcnow() description = res_type_info.DESCRIPTION.copy() description.update( stats_framework_gviz.get_description_key(resolution)) stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False) template_data = self.process_data(description, stats_data) template_data['duration'] = duration template_data['now'] = now template_data['resolution'] = resolution self.response.write( template.render(res_type_info.TEMPLATE, template_data))
def Stats(self, request, context): res = self._RESOLUTION_MAP.get(request.resolution) if not res: context.set_code(StatusCode.INVALID_ARGUMENT) context.set_details('Invalid resolution') return None if not 1 <= request.page_size <= 1000: context.set_code(StatusCode.INVALID_ARGUMENT) context.set_details( 'Invalid page_size; must be between 1 and 1000') return None if request.latest_time.seconds: now = request.latest_time.ToDatetime() else: now = utils.utcnow() # Round time to the minute. now = datetime.datetime(*now.timetuple()[:5], tzinfo=now.tzinfo) entities = stats_framework.get_stats(stats.STATS_HANDLER, res, now, request.page_size, False) out = isolated_pb2.StatsResponse() for s in entities: stats.snapshot_to_proto(s, out.measurements.add()) logging.info('Found %d entities', len(entities)) return out
def get_json(request, response, handler, resolution, description, order): """Returns the statistic data as a Google Visualization compatible reply. The return can be either JSON or JSONP, depending if the header 'X-DataSource-Auth' is set in the request. Note that this is not real JSON, as explained in developers.google.com/chart/interactive/docs/dev/implementing_data_source Exposes the data in the format described at https://developers.google.com/chart/interactive/docs/reference#dataparam and https://developers.google.com/chart/interactive/docs/querylanguage Arguments: - request: A webapp2.Request. - response: A webapp2.Response. - handler: A StatisticsFramework. - resolution: One of 'days', 'hours' or 'minutes'. - description: Dict describing the columns. - order: List describing the order to use for the columns. Raises: ValueError if a 400 should be returned. """ tqx_args = process_tqx(request.params.get('tqx', '')) duration = utils.get_request_as_int(request, 'duration', 120, 1, 256) now = utils.get_request_as_datetime(request, 'now') table = stats_framework.get_stats(handler, resolution, now, duration, True) return get_json_raw(request, response, table, description, order, tqx_args)
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats( stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def get(self2): self2.response.headers['Content-Type'] = ( 'application/json; charset=utf-8') duration = int(self2.request.get('duration', 120)) now = self2.request.get('now') resolution = self2.request.get('resolution') data = stats_framework.get_stats( self.h, resolution, now, duration, True) self2.response.write(stats_framework.utils.encode_to_json(data))
def send_response(self, res_type_info, resolution): if resolution not in stats_framework.RESOLUTIONS: self.abort(404) duration = utils.get_request_as_int(self.request, "duration", default=120, min_value=1, max_value=1000) now = utils.get_request_as_datetime(self.request, "now") description = res_type_info.DESCRIPTION.copy() description.update(stats_framework_gviz.get_description_key(resolution)) stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False) tqx_args = tqx_args = stats_framework_gviz.process_tqx(self.request.params.get("tqx", "")) try: stats_framework_gviz.get_json_raw( self.request, self.response, self.get_table(stats_data), description, res_type_info.ORDER, tqx_args ) except ValueError as e: self.abort(400, str(e))
def get(self): """Presents nice recent statistics. It fetches data from the 'JSON' API. """ # Preloads the data to save a complete request. resolution = self.request.params.get('resolution', 'hours') duration = utils.get_request_as_int(self.request, 'duration', 120, 1, 1000) description = _GVIZ_DESCRIPTION.copy() description.update(stats_framework_gviz.get_description_key(resolution)) table = stats_framework.get_stats( stats.STATS_HANDLER, resolution, None, duration, True) params = { 'duration': duration, 'initial_data': gviz_api.DataTable(description, table).ToJSon( columns_order=_GVIZ_COLUMNS_ORDER), 'now': datetime.datetime.utcnow(), 'resolution': resolution, } self.response.write(template.render('isolate/stats.html', params))
def send_response(self, res_type_info): """Presents nice recent statistics. It preloads data in the template for maximum responsiveness and interactively fetches data from the JSON API. """ # Preloads the data to save a complete request. resolution = self.request.params.get("resolution", "hours") if resolution not in stats_framework.RESOLUTIONS: self.abort(404) duration = utils.get_request_as_int(self.request, "duration", default=120, min_value=1, max_value=1000) now = utils.get_request_as_datetime(self.request, "now") now = now or datetime.datetime.utcnow() description = res_type_info.DESCRIPTION.copy() description.update(stats_framework_gviz.get_description_key(resolution)) stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False) template_data = self.process_data(description, stats_data) template_data["duration"] = duration template_data["now"] = now template_data["resolution"] = resolution self.response.write(template.render(res_type_info.TEMPLATE, template_data))
def send_response(self, res_type_info, resolution): if resolution not in stats_framework.RESOLUTIONS: self.abort(404) duration = utils.get_request_as_int(self.request, 'duration', default=120, min_value=1, max_value=1000) now = utils.get_request_as_datetime(self.request, 'now') description = res_type_info.DESCRIPTION.copy() description.update( stats_framework_gviz.get_description_key(resolution)) stats_data = stats_framework.get_stats(stats.STATS_HANDLER, resolution, now, duration, False) tqx_args = tqx_args = stats_framework_gviz.process_tqx( self.request.params.get('tqx', '')) try: stats_framework_gviz.get_json_raw(self.request, self.response, self.get_table(stats_data), description, res_type_info.ORDER, tqx_args) except ValueError as e: self.abort(400, str(e))
def test_framework_last_few(self): called = [] def gen_data(start, end): """Returns fake statistics.""" self.assertEqual(start + 60, end) called.append(start) return Snapshot(requests=1, b=1, inner=InnerSnapshot(c='%d,' % len(called))) handler = stats_framework.StatisticsFramework('test_framework', Snapshot, gen_data) now = get_now() self.mock_now(now, 0) handler._set_last_processed_time( strip_seconds(now) - datetime.timedelta(seconds=3 * 60)) i = handler.process_next_chunk(1) self.assertEqual(2, i) self.assertEqual(1, stats_framework.StatsRoot.query().count()) self.assertEqual(1, handler.stats_day_cls.query().count()) self.assertEqual(1, handler.stats_hour_cls.query().count()) self.assertEqual(2, handler.stats_minute_cls.query().count()) root = handler.root_key.get() self.assertEqual( strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp) # Trying to process more won't do anything. i = handler.process_next_chunk(1) self.assertEqual(0, i) root = handler.root_key.get() self.assertEqual( strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp) expected = [ { 'key': '2010-01-02', 'requests': 0, 'b': 0.0, 'd': [], 'inner': { 'c': u'' }, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'days', now, 100, True)) expected = [ { 'key': '2010-01-02T03', 'requests': 2, 'b': 2.0, 'd': [], 'inner': { 'c': u'1,2,' }, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'hours', now, 100, True)) expected = [ { 'key': '2010-01-02T03:03', 'requests': 1, 'b': 1.0, 'd': [], 'inner': { 'c': u'2,' }, }, { 'key': '2010-01-02T03:02', 'requests': 1, 'b': 1.0, 'd': [], 'inner': { 'c': u'1,' }, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'minutes', now, 100, True))
def test_framework_last_few(self): called = [] def gen_data(start, end): """Returns fake statistics.""" self.assertEqual(start + 60, end) called.append(start) return Snapshot( requests=1, b=1, inner=InnerSnapshot(c='%d,' % len(called))) handler = stats_framework.StatisticsFramework( 'test_framework', Snapshot, gen_data) now = get_now() self.mock_now(now, 0) handler._set_last_processed_time( strip_seconds(now) - datetime.timedelta(seconds=3*60)) i = handler.process_next_chunk(1) self.assertEqual(2, i) self.assertEqual(1, stats_framework.StatsRoot.query().count()) self.assertEqual(1, handler.stats_day_cls.query().count()) self.assertEqual(1, handler.stats_hour_cls.query().count()) self.assertEqual(2, handler.stats_minute_cls.query().count()) root = handler.root_key.get() self.assertEqual( strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp) # Trying to process more won't do anything. i = handler.process_next_chunk(1) self.assertEqual(0, i) root = handler.root_key.get() self.assertEqual( strip_seconds(now) - datetime.timedelta(seconds=60), root.timestamp) expected = [ { 'key': now.date(), 'requests': 0, 'b': 0.0, 'd': [], 'inner': {'c': u''}, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'days', now, 100, True)) expected = [ { 'key': datetime.datetime(*now.timetuple()[:4]), 'requests': 2, 'b': 2.0, 'd': [], 'inner': {'c': u'1,2,'}, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'hours', now, 100, True)) expected = [ { 'key': datetime.datetime( *(now - datetime.timedelta(seconds=60)).timetuple()[:5]), 'requests': 1, 'b': 1.0, 'd': [], 'inner': {'c': u'2,'}, }, { 'key': datetime.datetime( *(now - datetime.timedelta(seconds=120)).timetuple()[:5]), 'requests': 1, 'b': 1.0, 'd': [], 'inner': {'c': u'1,'}, }, ] self.assertEqual( expected, stats_framework.get_stats(handler, 'minutes', now, 100, True))