class VarnishPurgerTest(TestCase):
@patch('http.client.HTTPConnection.getresponse',
Mock(return_value=Mock(status=200)))
@patch('http.client.HTTPConnection.request')
def test_should_purge_active_varnishes(self, response_mock):
expected_message = "varnish http response code: 200, url=http://example.com/modified_resource/"
servers = [
VarnishServer(ip='127.0.0.1'),
VarnishServer(ip='127.0.0.2')
]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/', servers)
self.assertEqual(2, len(response['success']))
self.assertEqual(expected_message, response['success']['127.0.0.1'])
@patch('http.client.HTTPConnection.request',
Mock(side_effect=BadStatusLine('bad status line')))
def test_should_report_error_on_bad_status_line(self):
expected_error = "Bad status line from varnish server, url=http://example.com/modified_resource/"
servers = [VarnishServer(ip='127.0.0.3')]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/', servers)
self.assertEqual(1, len(response['error']))
self.assertEqual(expected_error, response['error']['127.0.0.3'])
@patch('http.client.HTTPConnection.request',
Mock(side_effect=Exception('Unknown error')))
def test_should_report_error_on_generic_exception(self):
expected_error = "Unexpected error: Unknown error, url=http://example.com/modified_resource/"
servers = [VarnishServer(ip='127.0.0.4')]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/', servers)
self.assertEqual(1, len(response['error']))
self.assertEqual(expected_error, response['error']['127.0.0.4'])
def test_do_http_with_bad_status_line(self, mock_get_connection):
mock_conn = mock_get_connection.return_value = Mock()
# First attempt: Error, second attempt: successful response
mock_response = Mock()
mock_conn.getresponse.side_effect = [BadStatusLine(0), mock_response]
# Stops at the fourth read call
mock_response.read.return_value = b"response data"
mock_response.status = 200
with patch('time.sleep'):
resp, body = self.connection.do_http('POST', '/rest/test', 'body')
self.assertEqual(body, 'response data')
mock_conn.request.assert_called_with(
'POST', '/rest/test', 'body', {
'Content-Type': 'application/json',
'X-API-Version': 300,
'Accept': 'application/json'
})
mock_conn.close.assert_has_calls([call(), call()])
def makeRequest(self,
uri=None,
data=None,
verb='GET',
incoming_headers=None,
encoder=True,
decoder=True,
contentType=None):
# METAL \m/
raise BadStatusLine(666)
def test_not_found_variant(self, urlopen, Request):
excp = BadStatusLine('')
urlopen.return_value.read.side_effect = excp
Request.get_host.return_value = 'archlinux.org'
Request.type.return_value = 'https'
call_command('mirrorcheck')
mirrorlog = MirrorLog.objects.first()
self.assertIn('Exception in processing', mirrorlog.error)
self.assertEqual(mirrorlog.is_success, False)
def read_headers(self):
status_line = yield from self.getline()
status_parts = status_line.split(None, 2)
if len(status_parts) != 3:
raise BadStatusLine(status_line)
self.http_version, status, self.reason = status_parts
self.status = int(status)
while True:
header_line = yield from self.getline()
if not header_line:
break
# TODO: Continuation lines.
key, value = header_line.split(':', 1)
self.headers.append((key, value.strip()))
def read_headers(self) -> Generator[Any, None, None]:
"""Read the response status and the request headers."""
status_line = yield from self.getline()
status_parts = status_line.split(None, 2)
if len(status_parts) != 3:
self.log(0, 'bad status_line', repr(status_line))
raise BadStatusLine(status_line)
self.http_version, status, self.reason = status_parts
self.status = int(status)
while True:
header_line = yield from self.getline()
if not header_line:
break
# TODO: Continuation lines.
key, value = header_line.split(':', 1)
self.headers.append((key, value.strip()))
def _get_ec2_user_data():
"""
Recursive call in _get_ec2_hostinfo() does not retrieve user-data.
"""
response = _call_aws("/latest/user-data")
# _call_aws returns None for all non '200' reponses,
# catching that here would rule out AWS resource
if response.status == 200:
response_data = response.read()
try:
return json.loads(response_data)
except ValueError:
return response_data
elif response.status == 404:
return ''
else:
raise BadStatusLine("Could not read EC2 user-data")
def _get_ec2_is_vpc():
"""
Determine if this instance is in a VPC or not
"""
resp = _call_aws('/latest/meta-data/mac')
mac = resp.read().decode('utf-8')
resp = _call_aws('/latest/meta-data/network/interfaces/macs/' + mac +
'/vpc-id')
if resp.status == 200:
return True
elif resp.status == 404:
return False
else:
raise BadStatusLine("Could not determine if instance is in VPC")
def test_download_to_stream_when_status_ok(self, mock_get_conn, mock_sleep):
mock_conn = Mock()
# First attempt: Error, second attempt: successful connection
mock_get_conn.side_effect = [BadStatusLine(0), mock_conn]
mock_response = mock_conn.getresponse.return_value
# Stops at the fourth read call
mock_response.read.side_effect = ['111', '222', '333', None]
mock_response.status = 200
mock_stream = Mock()
result = self.connection.download_to_stream(mock_stream, '/rest/download.zip',
custom_headers={'custom': 'custom'})
self.assertTrue(result)
mock_stream.write.assert_has_calls([call('111'), call('222'), call('333')])
def _get_ec2_additional():
"""
Recursive call in _get_ec2_hostinfo() does not retrieve some of
the hosts information like region, availability zone or
architecture.
"""
response = _call_aws("/latest/dynamic/instance-identity/document")
# _call_aws returns None for all non '200' reponses,
# catching that here would rule out AWS resource
if response.status == 200:
response_data = response.read()
try:
data = json.loads(response_data.decode('utf-8'))
except ValueError as e:
data = {}
data = _snake_caseify_dict(data)
data.update({'instance_identity': {'document': response_data}})
return data
else:
raise BadStatusLine("Could not read EC2 metadata")
def read_response(conn):
"""Read an HTTP response from a connection."""
@asyncio.coroutine
def getline():
line = (yield from conn.reader.readline()).decode('latin-1').rstrip()
logger.info('< %s', line)
return line
status_line = yield from getline()
status_parts = status_line.split(None, 2)
if len(status_parts) != 3 or not status_parts[1].isdigit():
logger.error('bad status_line %r', status_line)
raise BadStatusLine(status_line)
http_version, status, reason = status_parts
status = int(status)
headers = {}
while True:
header_line = yield from getline()
if not header_line:
break
key, value = header_line.split(':', 1)
# TODO: Continuation lines; multiple header lines per key..
headers[key.lower()] = value.lstrip()
if 'content-length' in headers:
nbytes = int(headers['content-length'])
output = asyncio.StreamReader()
asyncio. async (length_handler(nbytes, conn.reader, output))
elif headers.get('transfer-encoding') == 'chunked':
output = asyncio.StreamReader()
asyncio. async (chunked_handler(conn.reader, output))
else:
output = conn.reader
return http_version[5:], status, reason, headers, output
class VarnishPurgerTest(TestCase):
@patch('http.client.HTTPConnection.getresponse',
Mock(return_value=Mock(status=200)))
@patch('http.client.HTTPConnection.request')
def test_should_purge_active_varnishes(self, request_mock):
expected_message_varnish1 = [
"varnish http response code: 200, url=http://example.com/modified_resource/, headers=[('Host', 'example.com'), ('header1', 'value1')], server=127.0.0.1:80", # noqa
"varnish http response code: 200, url=http://example.com/modified_resource/, headers=[('Host', 'example.com'), ('header1', 'value2')], server=127.0.0.1:80" # noqa
]
expected_message_varnish2 = [
"varnish http response code: 200, url=http://example.com/modified_resource/, headers=[('Host', 'example.com'), ('header1', 'value1')], server=127.0.0.2:80", # noqa
"varnish http response code: 200, url=http://example.com/modified_resource/, headers=[('Host', 'example.com'), ('header1', 'value2')], server=127.0.0.2:80" # noqa
]
servers = [
VarnishServer(ip='127.0.0.1'),
VarnishServer(ip='127.0.0.2')
]
result = VarnishPurger().purge_url(
'http://example.com/modified_resource/',
servers,
headers={'header1': ['value1', 'value2']}) # noqa
calls = [
call('PURGE',
'/modified_resource/',
body='',
headers={
'header1': 'value1',
'Host': 'example.com'
}),
call('PURGE',
'/modified_resource/',
body='',
headers={
'header1': 'value2',
'Host': 'example.com'
}),
call('PURGE',
'/modified_resource/',
body='',
headers={
'header1': 'value1',
'Host': 'example.com'
}),
call('PURGE',
'/modified_resource/',
body='',
headers={
'header1': 'value2',
'Host': 'example.com'
})
]
request_mock.assert_has_calls(calls, any_order=True)
self.assertEqual(2, len(result['success']))
self.assertCountEqual(expected_message_varnish1,
result['success']['127.0.0.1'])
self.assertCountEqual(expected_message_varnish2,
result['success']['127.0.0.2'])
@patch('http.client.HTTPConnection.getresponse',
Mock(return_value=Mock(status=200)))
@patch('http.client.HTTPConnection.request')
def test_should_provided_host_header_should_be_passed_tu_purged_server(
self, request_mock):
servers = [VarnishServer(ip='127.0.0.1')]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/',
servers,
headers={'Host': ['foo.com']}) # noqa
request_mock.assert_called_with('PURGE',
'/modified_resource/',
body='',
headers={'Host': 'foo.com'})
@patch('http.client.HTTPConnection.request',
Mock(side_effect=BadStatusLine('bad status line')))
def test_should_report_error_on_bad_status_line(self):
expected_error = [
"Bad status line from varnish server, url=http://example.com/modified_resource/, headers=[('Host', 'example.com')], server=127.0.0.3:80" # noqa
]
servers = [VarnishServer(ip='127.0.0.3')]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/', servers)
self.assertEqual(1, len(response['error']))
self.assertEqual(expected_error, response['error']['127.0.0.3'])
@patch('http.client.HTTPConnection.request',
Mock(side_effect=Exception('Unknown error')))
def test_should_report_error_on_generic_exception(self):
expected_error = [
"Unexpected error: Unknown error, url=http://example.com/modified_resource/, headers=[('Host', 'example.com')], server=127.0.0.4:80" # noqa
]
servers = [VarnishServer(ip='127.0.0.4')]
response = VarnishPurger().purge_url(
'http://example.com/modified_resource/', servers)
self.assertEqual(1, len(response['error']))
self.assertEqual(expected_error, response['error']['127.0.0.4'])
def test_should_prepare_headers_combinations(self):
excepted_result = [{
'header2': 'val1',
'header1': 'val1'
}, {
'header2': 'val1',
'header1': 'val2'
}, {
'header2': 'val2',
'header1': 'val1'
}, {
'header2': 'val2',
'header1': 'val2'
}]
headers = {"header1": ["val1", "val2"], "header2": ["val1", "val2"]}
result = VarnishPurger().prepare_headers_combinations(headers)
self.assertCountEqual(result, excepted_result)