def test_response_time(self):
"""Tests that the response time we passed is returned."""
http_metrics = metrics_utils.extract_http_metrics('', '', 0, 0.25)
self.assertEqual(0.25, http_metrics['response_time'])
http_metrics = metrics_utils.extract_http_metrics('', '', 0, 12345.25)
self.assertEqual(12345.25, http_metrics['response_time'])
def test_extracts_path_changes(self):
"""Tests that we extract paths for /changes/."""
# /changes/<change-id>
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/proj%2Fsrc%7Emaster%7EI1234abcd',
'', 0, 0)
self.assertEqual('changes', http_metrics['path'])
# /changes/?q=<something>
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/?q=owner:me+OR+cc:me',
'', 0, 0)
self.assertEqual('changes', http_metrics['path'])
# /changes/#<something>
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/#something',
'', 0, 0)
self.assertEqual('changes', http_metrics['path'])
# /changes/<change-id>/<anything> does not map to changes.
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/12345678/message',
'', 0, 0)
self.assertNotEqual('changes', http_metrics['path'])
def test_status(self):
"""Tests that the response status we passed is returned."""
http_metrics = metrics_utils.extract_http_metrics('', '', 123, 0)
self.assertEqual(123, http_metrics['status'])
http_metrics = metrics_utils.extract_http_metrics('', '', 404, 0)
self.assertEqual(404, http_metrics['status'])
def test_validates_method(self):
"""Test that we validate the HTTP method used."""
# Regular case
http_metrics = metrics_utils.extract_http_metrics('', 'POST', 0, 0)
self.assertEqual('POST', http_metrics['method'])
# Unexpected method is not reported
http_metrics = metrics_utils.extract_http_metrics('', 'DEMAND', 0, 0)
self.assertNotIn('method', http_metrics)
def test_extracts_host(self):
"""Test that we extract the host from the requested URI."""
# Regular case
http_metrics = metrics_utils.extract_http_metrics(
'https://chromium-review.googlesource.com/foo/bar?q=baz', '', 0, 0)
self.assertEqual('chromium-review.googlesource.com', http_metrics['host'])
# Unexpected host
http_metrics = metrics_utils.extract_http_metrics(
'https://foo-review.googlesource.com/', '', 0, 0)
self.assertNotIn('host', http_metrics)
def test_extracts_path(self):
"""Test that we extract the matching path from the requested URI."""
# Regular case
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/1234/revisions/deadbeef/commit',
'', 0, 0)
self.assertEqual('changes/revisions/commit', http_metrics['path'])
# No matching paths
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/changes/1234/unexpected/path', '', 0, 0)
self.assertNotIn('path', http_metrics)
def test_extracts_arguments(self):
"""Test that we can extract arguments from the requested URI."""
# Regular case
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/?q=123&foo=bar&o=ALL_REVISIONS', '', 0, 0)
self.assertEqual(['ALL_REVISIONS'], http_metrics['arguments'])
# Some unexpected arguments are filtered out.
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/?o=ALL_REVISIONS&o=LABELS&o=UNEXPECTED',
'', 0, 0)
self.assertEqual(['ALL_REVISIONS', 'LABELS'], http_metrics['arguments'])
# No valid arguments, so arguments is not present
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/?o=bar&baz=1', '', 0, 0)
self.assertNotIn('arguments', http_metrics)
# No valid arguments, so arguments is not present
http_metrics = metrics_utils.extract_http_metrics(
'https://review.example.com/?foo=bar&baz=1', '', 0, 0)
self.assertNotIn('arguments', http_metrics)
def ReadHttpResponse(conn, accept_statuses=frozenset([200])):
"""Reads an HTTP response from a connection into a string buffer.
Args:
conn: An Http object created by CreateHttpConn above.
accept_statuses: Treat any of these statuses as success. Default: [200]
Common additions include 204, 400, and 404.
Returns: A string buffer containing the connection's reply.
"""
sleep_time = 1.5
for idx in range(TRY_LIMIT):
before_response = time.time()
response, contents = conn.request(**conn.req_params)
contents = contents.decode('utf-8', 'replace')
response_time = time.time() - before_response
metrics.collector.add_repeated(
'http_requests',
metrics_utils.extract_http_metrics(
conn.req_params['uri'], conn.req_params['method'], response.status,
response_time))
# If response.status is an accepted status,
# or response.status < 500 then the result is final; break retry loop.
# If the response is 404/409 it might be because of replication lag,
# so keep trying anyway.
if (response.status in accept_statuses
or response.status < 500 and response.status not in [404, 409]):
LOGGER.debug('got response %d for %s %s', response.status,
conn.req_params['method'], conn.req_params['uri'])
# If 404 was in accept_statuses, then it's expected that the file might
# not exist, so don't return the gitiles error page because that's not
# the "content" that was actually requested.
if response.status == 404:
contents = ''
break
# A status >=500 is assumed to be a possible transient error; retry.
http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
LOGGER.warn('A transient error occurred while querying %s:\n'
'%s %s %s\n'
'%s %d %s',
conn.req_host, conn.req_params['method'],
conn.req_params['uri'],
http_version, http_version, response.status, response.reason)
if idx < TRY_LIMIT - 1:
LOGGER.info('Will retry in %d seconds (%d more times)...',
sleep_time, TRY_LIMIT - idx - 1)
time_sleep(sleep_time)
sleep_time = sleep_time * 2
# end of retries loop
if response.status in accept_statuses:
return StringIO(contents)
if response.status in (302, 401, 403):
www_authenticate = response.get('www-authenticate')
if not www_authenticate:
print('Your Gerrit credentials might be misconfigured.')
else:
auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
host = auth_match.group(1) if auth_match else conn.req_host
print('Authentication failed. Please make sure your .gitcookies '
'file has credentials for %s.' % host)
print('Try:\n git cl creds-check')
reason = '%s: %s' % (response.reason, contents)
raise GerritError(response.status, reason)
def ReadHttpResponse(conn, accept_statuses=frozenset([200])):
"""Reads an HTTP response from a connection into a string buffer.
Args:
conn: An Http object created by CreateHttpConn above.
accept_statuses: Treat any of these statuses as success. Default: [200]
Common additions include 204, 400, and 404.
Returns: A string buffer containing the connection's reply.
"""
sleep_time = 1.5
for idx in range(TRY_LIMIT):
before_response = time.time()
response, contents = conn.request(**conn.req_params)
response_time = time.time() - before_response
metrics.collector.add_repeated(
'http_requests',
metrics_utils.extract_http_metrics(conn.req_params['uri'],
conn.req_params['method'],
response.status, response_time))
# Check if this is an authentication issue.
www_authenticate = response.get('www-authenticate')
if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND)
and www_authenticate):
auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
host = auth_match.group(1) if auth_match else conn.req_host
reason = (
'Authentication failed. Please make sure your .gitcookies file '
'has credentials for %s' % host)
raise GerritAuthenticationError(response.status, reason)
# If response.status < 500 then the result is final; break retry loop.
# If the response is 404/409, it might be because of replication lag, so
# keep trying anyway.
if ((response.status < 500 and response.status not in [404, 409])
or response.status in accept_statuses):
LOGGER.debug('got response %d for %s %s', response.status,
conn.req_params['method'], conn.req_params['uri'])
# If 404 was in accept_statuses, then it's expected that the file might
# not exist, so don't return the gitiles error page because that's not
# the "content" that was actually requested.
if response.status == 404:
contents = ''
break
# A status >=500 is assumed to be a possible transient error; retry.
http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
LOGGER.warn(
'A transient error occurred while querying %s:\n'
'%s %s %s\n'
'%s %d %s', conn.req_host, conn.req_params['method'],
conn.req_params['uri'], http_version, http_version,
response.status, response.reason)
if response.status == 404:
# TODO(crbug/881860): remove this hack.
# HACK: try different Gerrit mirror as a workaround for potentially
# out-of-date mirror hit through default routing.
if conn.req_host == 'chromium-review.googlesource.com':
conn.req_params['uri'] = _UseGerritMirror(
conn.req_params['uri'], 'chromium-review.googlesource.com')
# And don't increase sleep_time in this case, since we suspect we've
# just asked wrong git mirror before.
sleep_time /= 2.0
if TRY_LIMIT - idx > 1:
LOGGER.info('Will retry in %d seconds (%d more times)...',
sleep_time, TRY_LIMIT - idx - 1)
time.sleep(sleep_time)
sleep_time = sleep_time * 2
# end of retries loop
if response.status not in accept_statuses:
if response.status in (401, 403):
print('Your Gerrit credentials might be misconfigured. Try: \n'
' git cl creds-check')
reason = '%s: %s' % (response.reason, contents)
raise GerritError(response.status, reason)
return StringIO(contents)