def _extract_snapshot_from_logs(start_time, end_time):
"""Returns a _Snapshot from the processed logs for the specified interval.
The data is retrieved from logservice via stats_framework.
"""
snapshot = _Snapshot()
total_lines = 0
parse_errors = 0
bots_active = {}
bots_inactive = {}
tasks_active = {}
for entry in stats_framework.yield_entries(start_time, end_time):
snapshot.http_requests += 1
if entry.request.status >= 500:
snapshot.http_failures += 1
for l in entry.entries:
if _parse_line(l, snapshot, bots_active, bots_inactive, tasks_active):
total_lines += 1
else:
parse_errors += 1
_post_process(snapshot, bots_active, bots_inactive, tasks_active)
logging.debug(
'_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
start_time, end_time, total_lines, parse_errors)
return snapshot
def generate_snapshot(start_time, end_time):
values = Snapshot()
for entry in stats_framework.yield_entries(start_time, end_time):
values.requests += 1
for l in entry.entries:
values.inner.c += l
return values
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None,
None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes',
self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def generate_snapshot(start_time, end_time):
values = Snapshot()
for entry in stats_framework.yield_entries(start_time, end_time):
values.requests += 1
for l in entry.entries:
values.inner.c += l
return values
def _extract_snapshot_from_logs(start_time, end_time):
"""Returns a _Snapshot from the processed logs for the specified interval.
The data is retrieved from logservice via stats_framework.
"""
snapshot = _Snapshot()
total_lines = 0
parse_errors = 0
bots_active = {}
bots_inactive = {}
tasks_active = {}
for entry in stats_framework.yield_entries(start_time, end_time):
snapshot.http_requests += 1
if entry.request.status >= 500:
snapshot.http_failures += 1
for l in entry.entries:
if _parse_line(l, snapshot, bots_active, bots_inactive,
tasks_active):
total_lines += 1
else:
parse_errors += 1
_post_process(snapshot, bots_active, bots_inactive, tasks_active)
logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
start_time, end_time, total_lines, parse_errors)
return snapshot
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None, None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(
stats.STATS_HANDLER, 'minutes', self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def test_yield_entries(self):
stats_framework_mock.reset_timestamp(self.h, self.now)
self.assertEqual(
0, len(list(stats_framework.yield_entries(None, None))))
self.assertEqual(
0, len(list(stats_framework.yield_entries(1, time.time()))))
self.assertEqual('Yay', self.app.get('/generate').body)
self.assertEqual(
1, len(list(stats_framework.yield_entries(None, None))))
self.assertEqual(
1, len(list(stats_framework.yield_entries(1, time.time()))))
self.assertEqual(
0, len(list(stats_framework.yield_entries(
None, utils.time_time()))))
def test_yield_entries(self):
stats_framework_mock.reset_timestamp(self.h, self.now)
self.assertEqual(0, len(list(stats_framework.yield_entries(None,
None))))
self.assertEqual(
0, len(list(stats_framework.yield_entries(1, time.time()))))
self.assertEqual('Yay', self.app.get('/generate').body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None,
None))))
self.assertEqual(
1, len(list(stats_framework.yield_entries(1, time.time()))))
self.assertEqual(
0, len(list(stats_framework.yield_entries(None,
utils.time_time()))))
def _extract_snapshot_from_logs(start_time, end_time):
"""Returns a _Snapshot from the processed logs for the specified interval.
The data is retrieved from logservice via stats_framework.
"""
values = _Snapshot()
total_lines = 0
parse_errors = 0
for entry in stats_framework.yield_entries(start_time, end_time):
values.requests += 1
if entry.request.status >= 400:
values.failures += 1
for l in entry.entries:
if _parse_line(l, values):
total_lines += 1
else:
parse_errors += 1
logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
start_time, end_time, total_lines, parse_errors)
return values
def _extract_snapshot_from_logs(start_time, end_time):
"""Returns a _Snapshot from the processed logs for the specified interval.
The data is retrieved from logservice via stats_framework.
"""
values = _Snapshot()
total_lines = 0
parse_errors = 0
for entry in stats_framework.yield_entries(start_time, end_time):
values.requests += 1
if entry.request.status >= 400:
values.failures += 1
for l in entry.entries:
if _parse_line(l, values):
total_lines += 1
else:
parse_errors += 1
logging.debug(
'_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
start_time, end_time, total_lines, parse_errors)
return values
def test_yield_empty(self):
self.testbed.init_modules_stub()
self.assertEqual(
0, len(list(stats_framework.yield_entries(None, None))))
def test_yield_empty(self):
self.testbed.init_modules_stub()
self.assertEqual(0, len(list(stats_framework.yield_entries(None,
None))))