Esempio n. 1
0
def _extract_snapshot_from_logs(start_time, end_time):
  """Returns a _Snapshot from the processed logs for the specified interval.

  The data is retrieved from logservice via stats_framework.
  """
  snapshot = _Snapshot()
  total_lines = 0
  parse_errors = 0
  bots_active = {}
  bots_inactive = {}
  tasks_active = {}

  for entry in stats_framework.yield_entries(start_time, end_time):
    snapshot.http_requests += 1
    if entry.request.status >= 500:
      snapshot.http_failures += 1

    for l in entry.entries:
      if _parse_line(l, snapshot, bots_active, bots_inactive, tasks_active):
        total_lines += 1
      else:
        parse_errors += 1

  _post_process(snapshot, bots_active, bots_inactive, tasks_active)
  logging.debug(
      '_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
      start_time, end_time, total_lines, parse_errors)
  return snapshot
Esempio n. 2
0
def generate_snapshot(start_time, end_time):
  values = Snapshot()
  for entry in stats_framework.yield_entries(start_time, end_time):
    values.requests += 1
    for l in entry.entries:
      values.inner.c += l
  return values
Esempio n. 3
0
    def _test_handler(self, url, added_data):
        stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)

        self.assertEqual('Yay', self.app.get(url).body)
        self.assertEqual(1, len(list(stats_framework.yield_entries(None,
                                                                   None))))

        self.mock_now(self.now, 60)
        self.assertEqual(10, stats.generate_stats())

        actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes',
                                           self.now, 1, True)
        expected = [
            {
                'contains_lookups': 0,
                'contains_requests': 0,
                'downloads': 0,
                'downloads_bytes': 0,
                'failures': 0,
                'key': datetime.datetime(2010, 1, 2, 3, 4),
                'other_requests': 0,
                'requests': 1,
                'uploads': 0,
                'uploads_bytes': 0,
            },
        ]
        expected[0].update(added_data)
        self.assertEqual(expected, actual)
Esempio n. 4
0
def generate_snapshot(start_time, end_time):
    values = Snapshot()
    for entry in stats_framework.yield_entries(start_time, end_time):
        values.requests += 1
        for l in entry.entries:
            values.inner.c += l
    return values
Esempio n. 5
0
def _extract_snapshot_from_logs(start_time, end_time):
    """Returns a _Snapshot from the processed logs for the specified interval.

  The data is retrieved from logservice via stats_framework.
  """
    snapshot = _Snapshot()
    total_lines = 0
    parse_errors = 0
    bots_active = {}
    bots_inactive = {}
    tasks_active = {}

    for entry in stats_framework.yield_entries(start_time, end_time):
        snapshot.http_requests += 1
        if entry.request.status >= 500:
            snapshot.http_failures += 1

        for l in entry.entries:
            if _parse_line(l, snapshot, bots_active, bots_inactive,
                           tasks_active):
                total_lines += 1
            else:
                parse_errors += 1

    _post_process(snapshot, bots_active, bots_inactive, tasks_active)
    logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
                  start_time, end_time, total_lines, parse_errors)
    return snapshot
Esempio n. 6
0
  def _test_handler(self, url, added_data):
    stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)

    self.assertEqual('Yay', self.app.get(url).body)
    self.assertEqual(1, len(list(stats_framework.yield_entries(None, None))))

    self.mock_now(self.now, 60)
    self.assertEqual(10, stats.generate_stats())

    actual = stats_framework.get_stats(
        stats.STATS_HANDLER, 'minutes', self.now, 1, True)
    expected = [
      {
        'contains_lookups': 0,
        'contains_requests': 0,
        'downloads': 0,
        'downloads_bytes': 0,
        'failures': 0,
        'key': datetime.datetime(2010, 1, 2, 3, 4),
        'other_requests': 0,
        'requests': 1,
        'uploads': 0,
        'uploads_bytes': 0,
      },
    ]
    expected[0].update(added_data)
    self.assertEqual(expected, actual)
Esempio n. 7
0
  def test_yield_entries(self):
    stats_framework_mock.reset_timestamp(self.h, self.now)

    self.assertEqual(
        0, len(list(stats_framework.yield_entries(None, None))))
    self.assertEqual(
        0, len(list(stats_framework.yield_entries(1, time.time()))))

    self.assertEqual('Yay', self.app.get('/generate').body)

    self.assertEqual(
        1, len(list(stats_framework.yield_entries(None, None))))
    self.assertEqual(
        1, len(list(stats_framework.yield_entries(1, time.time()))))
    self.assertEqual(
        0, len(list(stats_framework.yield_entries(
          None, utils.time_time()))))
Esempio n. 8
0
    def test_yield_entries(self):
        stats_framework_mock.reset_timestamp(self.h, self.now)

        self.assertEqual(0, len(list(stats_framework.yield_entries(None,
                                                                   None))))
        self.assertEqual(
            0, len(list(stats_framework.yield_entries(1, time.time()))))

        self.assertEqual('Yay', self.app.get('/generate').body)

        self.assertEqual(1, len(list(stats_framework.yield_entries(None,
                                                                   None))))
        self.assertEqual(
            1, len(list(stats_framework.yield_entries(1, time.time()))))
        self.assertEqual(
            0, len(list(stats_framework.yield_entries(None,
                                                      utils.time_time()))))
Esempio n. 9
0
def _extract_snapshot_from_logs(start_time, end_time):
    """Returns a _Snapshot from the processed logs for the specified interval.

  The data is retrieved from logservice via stats_framework.
  """
    values = _Snapshot()
    total_lines = 0
    parse_errors = 0
    for entry in stats_framework.yield_entries(start_time, end_time):
        values.requests += 1
        if entry.request.status >= 400:
            values.failures += 1
        for l in entry.entries:
            if _parse_line(l, values):
                total_lines += 1
            else:
                parse_errors += 1
    logging.debug('_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
                  start_time, end_time, total_lines, parse_errors)
    return values
Esempio n. 10
0
def _extract_snapshot_from_logs(start_time, end_time):
  """Returns a _Snapshot from the processed logs for the specified interval.

  The data is retrieved from logservice via stats_framework.
  """
  values = _Snapshot()
  total_lines = 0
  parse_errors = 0
  for entry in stats_framework.yield_entries(start_time, end_time):
    values.requests += 1
    if entry.request.status >= 400:
      values.failures += 1
    for l in entry.entries:
      if _parse_line(l, values):
        total_lines += 1
      else:
        parse_errors += 1
  logging.debug(
      '_extract_snapshot_from_logs(%s, %s): %d lines, %d errors',
      start_time, end_time, total_lines, parse_errors)
  return values
Esempio n. 11
0
 def test_yield_empty(self):
   self.testbed.init_modules_stub()
   self.assertEqual(
       0, len(list(stats_framework.yield_entries(None, None))))
Esempio n. 12
0
 def test_yield_empty(self):
     self.testbed.init_modules_stub()
     self.assertEqual(0, len(list(stats_framework.yield_entries(None,
                                                                None))))