def test_retry_successfull(self, mock_get_connection):
# this tests the read function when first HTTP request returns a 503 but the second one
# is successfull and returns a 200
# mock the 503 response
mockHttpResponse503 = Mock(name="HttpResponse503")
mockHttpResponse503.getheader.return_value = 1
mockHttpResponse503.status = 503
mockHttpResponse503.read.return_value = "Fail"
mockHttpResponse503.len.return_value = 10
# mock the 200 response
mockHttpResponse200 = Mock(name="HttpResponse200")
mockHttpResponse200.getheader.return_value = 1
mockHttpResponse200.status = 200
mockHttpResponse200.read.return_value = "Testing"
mockHttpResponse200.len.return_value = 10
conn = Connection()
mockHttpRequest = Mock(name="HttpRequest")
# set a 503 response first followed by a 200 response
mockHttpRequest.getresponse = MagicMock(name="getresponse mock")
generator = mockHttpRequest.getresponse.iter.return_value
iterator = iter([mockHttpResponse503, mockHttpResponse200])
generator.__iter__.return_value = iterator
conn.get_connection.return_value = mockHttpRequest
#vofile = vos.VOFile(["Some URL"], mockConn, "GET")
vofile = vos.VOFile(["Some URL"], conn, "GET")
def test_fail_max_retry(self):
# this tests the read function when HTTP requests keep returning 503s
# read call fails when it reaches the maximum number of retries, in
# this case set to 2
# mock the 503 responses
mock_resp = Mock(name="503 resp")
mock_resp.status_code = 503
mock_resp.content = "Testing"
headers = {
'Content-Length': 10,
'X-CADC-Content-Length': 5,
'Retry-After': 4
}
def getheader(name, default):
return headers[name]
mock_resp.headers = MagicMock()
mock_resp.headers.get.side_effect = getheader
mock_resp.text = 'Try again later'
conn = Connection()
conn.session.send = Mock(return_value=mock_resp)
vofile = vos.VOFile(["Some URL"], conn, "GET")
req = requests.Request("GET", "http://some/url")
vofile.request = req.prepare()
# set number of retries to 1 and check the OSError was thrown
vofile.maxRetries = 1
with self.assertRaises(OSError) as cm:
vofile.read()
mock_resp.headers.get.assert_called_with('Retry-After', 5)
def test_retry_412_successfull(self, mock_get_connection):
# this tests the read function when first HTTP request returns a
# 412 but the second one
# is successful and returns a 200
# mock the 412 response
mockHttpResponse412 = Mock(name="HttpResponse412")
mockHttpResponse412.getheader.return_value = 1
mockHttpResponse412.status = 412
mockHttpResponse412.read.return_value = "Fail"
# mock the 200 response
mockHttpResponse200 = Mock(name="HttpResponse200")
mockHttpResponse200.getheader.return_value = 1
mockHttpResponse200.status = 200
mockHttpResponse200.read.return_value = "Testing"
conn = Connection(resource_id='ivo://cadc.nrc.ca/vault')
mockHttpRequest = Mock(name="HttpRequest")
# set a 412 response first followed by a 200 response
mockHttpRequest.getresponse = MagicMock(
side_effect=[mockHttpResponse412, mockHttpResponse200])
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(["Some URL"], conn, "GET")
vofile.currentRetryDelay = 2
def test_move(self):
mock_resp_403 = Mock(name="mock_resp_303")
mock_resp_403.status_code = 403
conn = Connection()
conn.session.post = Mock(return_value=mock_resp_403)
client = Client(conn=conn)
uri1 = 'notvos://cadc.nrc.ca!vospace/nosuchfile1'
uri2 = 'notvos://cadc.nrc.ca!vospace/nosuchfile2'
with self.assertRaises(OSError):
client.move(uri1, uri2)
def test_seek(self):
my_mock = MagicMock()
with patch('vos.VOFile.open', my_mock):
url_list = ['http://foo.com']
conn = Connection()
method = 'GET'
vofile = VOFile(url_list, conn, method, size=25)
vofile.seek(10, os.SEEK_CUR)
self.assertEquals(10, vofile._fpos)
vofile.seek(5, os.SEEK_SET)
self.assertEquals(5, vofile._fpos)
vofile.seek(10, os.SEEK_END)
self.assertEquals(15, vofile._fpos)
def test_checkstatus(self, mock_get_connection):
# Verify the md5sum and size are extracted from the HTTP header
conn = Connection()
# test successful - use first url
vofile = vos.VOFile(None, conn, "GET")
mock_resp = Object
mock_resp.status_code = 200
mock_resp.headers = {
'Content-MD5': 12345,
'Content-Length': 10,
'X-CADC-Content-Length': 10
}
vofile.resp = mock_resp
self.assertTrue(vofile.checkstatus())
self.assertEqual(vofile.get_file_info(), (10, 12345))
def test_checkstatus(self, mock_get_connection):
# Verify the md5sum and size are extracted from the HTTP header
conn = Connection()
# test successful - use first url
vofile = vos.VOFile(None, conn, "GET")
mockHttpResponse200 = Mock(name="HttpResponse200")
mockHttpResponse200.getheader = Mock(side_effect=SideEffect(
{
('Content-MD5', None): 12345,
('Content-Length', 0): 10,
},
name="mockHttpResponse200.getheader"))
mockHttpResponse200.status = 200
vofile.resp = mockHttpResponse200
self.assertTrue(vofile.checkstatus())
self.assertEqual(vofile.getFileInfo(), (10, 12345))
def test_open(self):
# Invalid mode raises OSError
with self.assertRaises(OSError):
client = Client()
client.open('vos://foo/bar', mode=-1)
with self.assertRaises(OSError):
client = Client()
client.get_node_url = Mock(return_value=None)
client.open(None, url=None)
conn = Connection()
mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
client = Client()
client.get_node_url = Mock(return_value=mock_vofile)
vofile = client.open(None, url=None)
self.assertEquals(vofile.url.URLs[0], 'http://foo.com/bar')
def test_fail_max_retry(self, mock_get_connection):
# this tests the read function when HTTP requests keep returning 503s
# read call fails when it reaches the maximum number of retries, in this case set to 2
# mock the 503 responses
mockHttpResponse = Mock(name="HttpResponse")
mockHttpResponse.getheader.return_value = 1 # try again after 1 sec
mockHttpResponse.status = 503
mockHttpResponse.read.return_value = "Testing"
conn = Connection()
mockHttpRequest = Mock(name="HttpRequest")
mockHttpRequest.getresponse.return_value = mockHttpResponse
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(["Some URL"], conn, "GET")
# set number of retries to 1 and check the IOError was thrown
vofile.maxRetries = 1
with self.assertRaises(IOError) as cm:
vofile.read()
mockHttpResponse.getheader.assert_called_with("Retry-After", 5)
def test_open(self):
# Invalid mode raises OSError
with self.assertRaises(OSError):
client = Client()
client.open('vos://foo/bar', mode=-10000)
with self.assertRaises(OSError):
client = Client()
client.get_node_url = Mock(return_value=None)
client.open(None, url=None)
conn = Connection(resource_id='ivo://cadc.nrc.ca/vault')
mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
client = Client()
client.get_node_url = Mock(return_value=mock_vofile)
endpoints_mock = Mock(conn=conn)
client.get_endpoints = Mock(return_value=endpoints_mock)
vofile = client.open(None, url=None)
self.assertEqual(vofile.url.URLs[0], 'http://foo.com/bar')
def test_multiple_urls(self, mock_get_connection):
transferURLs = ['http://url1.ca', 'http://url2.ca', 'http://url3.ca']
# mock the 503 response
mockHttpResponse503 = Mock(name="HttpResponse503")
mockHttpResponse503.getheader.return_value = 3
mockHttpResponse503.status = 503
mockHttpResponse503.read.return_value = "Try again"
# mock the 200 response
mockHttpResponse200 = Mock(name="HttpResponse200")
mockHttpResponse200.getheader.return_value = 1
mockHttpResponse200.status = 200
mockHttpResponse200.read.return_value = "Testing"
# mock the 412 response
mockHttpResponse404 = Mock(name="HttpResponse404")
mockHttpResponse404.getheader.return_value = 1
mockHttpResponse404.status = 404
mockHttpResponse404.read.return_value = "Fail"
conn = Connection()
mockHttpRequest = Mock(name="HttpRequest")
# test successful - use first url
self.responses = [mockHttpResponse200]
mockHttpRequest.getresponse = Mock(side_effect=self.side_effect)
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(transferURLs, conn, "GET")
vofile.read()
assert (vofile.url == transferURLs[0])
assert (vofile.urlIndex == 0)
assert (len(vofile.URLs) == 3)
# test first url busy
self.responses = [mockHttpResponse503, mockHttpResponse200]
mockHttpRequest.getresponse = Mock(side_effect=self.side_effect)
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(transferURLs, conn, "GET")
vofile.read()
assert (vofile.url == transferURLs[1])
assert (vofile.urlIndex == 1)
assert (len(vofile.URLs) == 3)
#test first url error - ignored internally, second url busy, third url works
self.responses = [
mockHttpResponse404, mockHttpResponse503, mockHttpResponse200
]
mockHttpRequest.getresponse = Mock(side_effect=self.side_effect)
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(transferURLs, conn, "GET")
vofile.read()
assert (vofile.url == transferURLs[2])
assert (vofile.urlIndex == 1)
assert (len(vofile.URLs) == 2)
# all urls busy first time, first one successfull second time
self.responses = [
mockHttpResponse503, mockHttpResponse503, mockHttpResponse503,
mockHttpResponse200
]
mockHttpRequest.getresponse = Mock(side_effect=self.side_effect)
conn.get_connection.return_value = mockHttpRequest
vofile = vos.VOFile(transferURLs, conn, "GET")
vofile.read()
assert (vofile.url == transferURLs[0])
assert (vofile.urlIndex == 0)
assert (len(vofile.URLs) == 3)
assert (3 == vofile.totalRetryDelay)
assert (1 == vofile.retries)
def test_multiple_urls(self):
transfer_urls = ['http://url1.ca', 'http://url2.ca', 'http://url3.ca']
# mock the 200 response
mock_resp_200 = requests.Response()
mock_resp_200.status_code = 200
mock_resp_200.headers = {'Content-Length': 10}
mock_resp_200.raw = Mock(read=Mock(return_value="abcd"))
# mock the 404 response
mock_resp_404 = requests.Response()
mock_resp_404.status_code = 404
mock_resp_404.headers = {'Content-Length': 10}
# mock the 503 response
mock_resp_503 = requests.Response()
mock_resp_503.status_code = 503
mock_resp_503.headers = {
'Content-Length': 10,
'Content-MD5': 12345,
'Retry-After': 1
}
conn = Connection()
# test successful - use first url
self.responses = [mock_resp_200]
vofile = vos.VOFile(transfer_urls, conn, "GET")
with patch('vos.vos.net.ws.Session.send',
Mock(side_effect=self.responses)):
vofile.read()
assert (vofile.url == transfer_urls[0])
assert (vofile.urlIndex == 0)
assert (len(vofile.URLs) == 3)
# test first url busy
self.responses = [mock_resp_503, mock_resp_200]
vofile = vos.VOFile(transfer_urls, conn, "GET")
with patch('vos.vos.net.ws.Session.send',
Mock(side_effect=self.responses)):
vofile.read()
assert (vofile.url == transfer_urls[1])
assert (vofile.urlIndex == 1)
assert (len(vofile.URLs) == 3)
# #test first url error - ignored internally, second url busy, third url works
# test 404 which raises OSError
self.responses = [mock_resp_404, mock_resp_503, mock_resp_200]
vofile = vos.VOFile(transfer_urls, conn, "GET")
#with self.assertRaises(exceptions.NotFoundException) as ex:
with patch('vos.vos.net.ws.requests.Session.send',
Mock(side_effect=self.responses)):
vofile.read()
assert (vofile.url == transfer_urls[2])
assert (vofile.urlIndex == 2)
assert (len(vofile.URLs) == 3)
# all urls busy first time, first one successful second time
self.responses = [
mock_resp_503, mock_resp_503, mock_resp_503, mock_resp_200
]
vofile = vos.VOFile(transfer_urls, conn, "GET")
with patch('vos.vos.net.ws.requests.Session.send',
Mock(side_effect=self.responses)):
vofile.read()
assert (vofile.url == transfer_urls[2])
assert (vofile.urlIndex == 2)
assert (len(vofile.URLs) == 3)