def _extract_snapshot_from_logs(start_time, end_time): """Returns a _Snapshot from the processed logs for the specified interval. The data is retrieved from logservice via stats_framework. """ snapshot = _Snapshot() total_lines = 0 parse_errors = 0 bots_active = {} bots_inactive = {} tasks_active = {} for entry in stats_framework.yield_entries(start_time, end_time): snapshot.http_requests += 1 if entry.request.status >= 500: snapshot.http_failures += 1 for l in entry.entries: if _parse_line(l, snapshot, bots_active, bots_inactive, tasks_active): total_lines += 1 else: parse_errors += 1 _post_process(snapshot, bots_active, bots_inactive, tasks_active) logging.debug( '_extract_snapshot_from_logs(%s, %s): %d lines, %d errors', start_time, end_time, total_lines, parse_errors) return snapshot
def generate_snapshot(start_time, end_time): values = Snapshot() for entry in stats_framework.yield_entries(start_time, end_time): values.requests += 1 for l in entry.entries: values.inner.c += l return values
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def _extract_snapshot_from_logs(start_time, end_time): """Returns a _Snapshot from the processed logs for the specified interval. The data is retrieved from logservice via stats_framework. """ snapshot = _Snapshot() total_lines = 0 parse_errors = 0 bots_active = {} bots_inactive = {} tasks_active = {} for entry in stats_framework.yield_entries(start_time, end_time): snapshot.http_requests += 1 if entry.request.status >= 500: snapshot.http_failures += 1 for l in entry.entries: if _parse_line(l, snapshot, bots_active, bots_inactive, tasks_active): total_lines += 1 else: parse_errors += 1 _post_process(snapshot, bots_active, bots_inactive, tasks_active) logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors', start_time, end_time, total_lines, parse_errors) return snapshot
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats( stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def test_yield_entries(self): stats_framework_mock.reset_timestamp(self.h, self.now) self.assertEqual( 0, len(list(stats_framework.yield_entries(None, None)))) self.assertEqual( 0, len(list(stats_framework.yield_entries(1, time.time())))) self.assertEqual('Yay', self.app.get('/generate').body) self.assertEqual( 1, len(list(stats_framework.yield_entries(None, None)))) self.assertEqual( 1, len(list(stats_framework.yield_entries(1, time.time())))) self.assertEqual( 0, len(list(stats_framework.yield_entries( None, utils.time_time()))))
def test_yield_entries(self): stats_framework_mock.reset_timestamp(self.h, self.now) self.assertEqual(0, len(list(stats_framework.yield_entries(None, None)))) self.assertEqual( 0, len(list(stats_framework.yield_entries(1, time.time())))) self.assertEqual('Yay', self.app.get('/generate').body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.assertEqual( 1, len(list(stats_framework.yield_entries(1, time.time())))) self.assertEqual( 0, len(list(stats_framework.yield_entries(None, utils.time_time()))))
def _extract_snapshot_from_logs(start_time, end_time): """Returns a _Snapshot from the processed logs for the specified interval. The data is retrieved from logservice via stats_framework. """ values = _Snapshot() total_lines = 0 parse_errors = 0 for entry in stats_framework.yield_entries(start_time, end_time): values.requests += 1 if entry.request.status >= 400: values.failures += 1 for l in entry.entries: if _parse_line(l, values): total_lines += 1 else: parse_errors += 1 logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors', start_time, end_time, total_lines, parse_errors) return values
def _extract_snapshot_from_logs(start_time, end_time): """Returns a _Snapshot from the processed logs for the specified interval. The data is retrieved from logservice via stats_framework. """ values = _Snapshot() total_lines = 0 parse_errors = 0 for entry in stats_framework.yield_entries(start_time, end_time): values.requests += 1 if entry.request.status >= 400: values.failures += 1 for l in entry.entries: if _parse_line(l, values): total_lines += 1 else: parse_errors += 1 logging.debug( '_extract_snapshot_from_logs(%s, %s): %d lines, %d errors', start_time, end_time, total_lines, parse_errors) return values
def test_yield_empty(self): self.testbed.init_modules_stub() self.assertEqual( 0, len(list(stats_framework.yield_entries(None, None))))
def test_yield_empty(self): self.testbed.init_modules_stub() self.assertEqual(0, len(list(stats_framework.yield_entries(None, None))))