def test_constructor(self):
# default client, patch netrc to ensure no netrc found
with patch('netrc.netrc.authenticators') as mock_authenticator:
mock_authenticator.side_effect = Exception('No netrc')
c = DataClient()
self.assertIsNone(c.certificate_file_location)
self.assertIsNone(c.basic_auth)
self.assertFalse(c.is_authorized)
self.assertEqual(c.protocol, 'http')
self.assertEqual(c.chunk_size, 1024)
self.assertIsNotNone(c.transfer_reader)
self.assertFalse(c.transfer_reader.validate)
self.assertIsNotNone(c.transfer_writer)
with patch('netrc.netrc.authenticators') as mock_authenticator:
mock_authenticator.side_effect = Exception('No netrc')
c = DataClient(schema_validate=True)
self.assertTrue(c.transfer_reader.validate)
def test_data_info_netrc(self,mock_session_head, mock_authenticator):
c = DataClient(certfile=test_cert, host=test_host)
self.assertTrue(c.is_authorized)
# a mocked response
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = {'key': 'value'}
mock_response.history = [1]
# Head request when we are using .netrc
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'http://%s/data/auth/%s/%s' % (test_host, test_archive,
test_localfile), verify=False )
mock_session_head.reset_mock()
def test_data_info_netrc(self, mock_session_head, mock_authenticator):
c = DataClient(certfile=test_cert, host=test_host)
self.assertTrue(c.is_authorized)
# a mocked response
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = {'key': 'value'}
mock_response.history = [1]
# Head request when we are using .netrc
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'http://%s/data/auth/%s/%s' %
(test_host, test_archive, test_localfile),
verify=False)
mock_session_head.reset_mock()
def test_data_info_cert(self, mock_session_head, mock_isfile):
c = DataClient(certfile=test_cert, host=test_host)
self.assertTrue(c.is_authorized)
# a mocked response
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = {'key': 'value'}
mock_response.history = [1]
# Head request when we have a cert
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'https://%s/data/pub/%s/%s' %
(test_host, test_archive, test_localfile),
verify=False)
mock_session_head.reset_mock()
mock_isfile.reset_mock()
# We can also handle the anonymous case here
c = DataClient(host=test_host, anonymous=True)
self.assertFalse(c.is_authorized)
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'http://%s/data/pub/%s/%s' %
(test_host, test_archive, test_localfile),
verify=False)
mock_session_head.reset_mock()
mock_isfile.reset_mock()
def test_data_info_cert(self,mock_session_head, mock_isfile):
c = DataClient(certfile=test_cert, host=test_host)
self.assertTrue(c.is_authorized)
# a mocked response
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = {'key': 'value'}
mock_response.history = [1]
# Head request when we have a cert
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'https://%s/data/pub/%s/%s' % (test_host, test_archive,
test_localfile), verify=False )
mock_session_head.reset_mock()
mock_isfile.reset_mock()
# We can also handle the anonymous case here
c = DataClient(host=test_host, anonymous=True)
self.assertFalse(c.is_authorized)
mock_session_head.return_value = mock_response
c.data_info(test_archive, test_localfile)
mock_session_head.assert_called_once_with(
'http://%s/data/pub/%s/%s' % (test_host, test_archive,
test_localfile), verify=False )
mock_session_head.reset_mock()
mock_isfile.reset_mock()
def test_get_file(self, mock_open, mock_requests_get, mock_session_post,
mock_isfile):
mock_file = MagicMock(spec=file)
mock_open.return_value = mock_file
c = DataClient(certfile=test_cert)
self.assertTrue(c.is_authorized)
#logger = c.get_logger(debug=True)
# a couple of mocked responses to requests
mock_response = Mock()
mock_response.text = test_get_transfer_resp_xml
mock_response.status_code = 200
mock_response.history = [1]
mock_response_get = Mock()
mock_response_get.text = test_get_transfer_resp_xml
mock_response_get.status_code = 200
mock_response_get.history = [1]
mock_response_get.iter_content = Mock(return_value=[1])
mock_response_get_fail = Mock()
mock_response_get_fail.text = test_get_transfer_resp_xml
mock_response_get_fail.status_code = 503
mock_response_get_fail.history = [1]
mock_response_get_fail.raise_for_status = Mock(
side_effect=Exception('Failed.'))
# We mock the client session POST response to return desired
# XML with a list of endpoints. We then check that the post
# was called with the correct values.
# The "mock_requests_get" is called to actually get the data,
# and we ensure it is called with the correct endpoint. It also
# requires an "iter_content" method that returns an iterable
# to simulate streaming.
mock_session_post.return_value = mock_response
mock_requests_get.return_value = mock_response_get
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_get.assert_called_once_with(test_endpoint1, stream=True)
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# First GET (endpoint1) returns a failed response. The second
# GET (endpoint2) will work
mock_requests_get.side_effect = [
mock_response_get_fail, mock_response_get
]
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_get.assert_has_calls([
call(test_endpoint1, stream=True),
call(test_endpoint2, stream=True)
])
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Both endpoints, and therefore the transfer, fail
mock_requests_get.side_effect = [
mock_response_get_fail, mock_response_get_fail
]
with self.assertRaises(TransferException):
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_get.assert_has_calls([
call(test_endpoint1, stream=True),
call(test_endpoint2, stream=True)
])
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
def test_put_file(self, mock_open, mock_requests_put, mock_session_post,
mock_isfile):
mock_file = MagicMock(spec=file)
mock_open.return_value = mock_file
c = DataClient(certfile=test_cert)
self.assertTrue(c.is_authorized)
#logger = c.get_logger(debug=True)
# a couple of mocked responses to requests
mock_response = Mock()
mock_response.text = test_put_transfer_resp_xml
mock_response.status_code = 200
mock_response.history = [1]
mock_response_fail = Mock()
mock_response_fail.text = test_put_transfer_resp_xml
mock_response_fail.status_code = 503
mock_response_fail.history = [1]
mock_response_fail.raise_for_status = Mock(
side_effect=Exception('Failed.'))
# make sure that transfer_file raises en error on bad inputs
with self.assertRaises(ValueError):
c.transfer_file(test_localfile)
# We mock the client session POST response to return desired
# XML with a list of endpoints. We then check that the post
# was called with the correct values.
# The "mock_requests_put" is called to actually put the data,
# and we ensure it is called with the correct endpoint.
#
# We do the whole thing three times: first time specifying a uri,
# second time specifying archive (filename for URI derived from
# local filename), and finally specifying both archive and filename.
for i in range(3):
mock_session_post.return_value = mock_response
mock_requests_put.return_value = mock_response
if i is 0:
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
elif i is 1:
c.transfer_file(test_localfile,
archive=test_archive,
is_put=True)
else:
c.transfer_file(test_localfile,
archive=test_archive,
filename=test_localfile,
is_put=True)
mock_session_post.assert_called_once_with(
c.base_url,
data=test_put_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_put.assert_called_once_with(test_endpoint1,
data=mock_file)
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Test specifying the stream
mock_session_post.return_value = mock_response
mock_requests_put.return_value = mock_response
c.transfer_file(test_localfile,
archive=test_archive,
filename=test_localfile,
is_put=True,
stream=test_stream)
expected_headers = copy.deepcopy(test_headers)
expected_headers['X-CADC-Stream'] = test_stream
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None,
verify=False,
headers=expected_headers)
mock_requests_put.assert_called_once_with(test_endpoint1,
data=mock_file)
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# First PUT (endpoint1) returns a failed response. The second
# PUT (endpoint2) will work
mock_requests_put.side_effect = [mock_response_fail, mock_response]
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_put.assert_has_calls([
call(test_endpoint1, data=mock_file),
call(test_endpoint2, data=mock_file)
])
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Both endpoints, and therefore the transfer, fail
mock_requests_put.side_effect = [
mock_response_fail, mock_response_fail
]
with self.assertRaises(TransferException):
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None,
verify=False,
headers=test_headers)
mock_requests_put.assert_has_calls([
call(test_endpoint1, data=mock_file),
call(test_endpoint2, data=mock_file)
])
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# An anonymous put should raise an UnauthorizedException
c.is_authorized = False
with self.assertRaises(UnauthorizedException):
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
def test_get_file(self,mock_open,mock_requests_get,mock_session_post,
mock_isfile):
mock_file = MagicMock(spec=file)
mock_open.return_value = mock_file
c = DataClient(certfile=test_cert)
self.assertTrue(c.is_authorized)
#logger = c.get_logger(debug=True)
# a couple of mocked responses to requests
mock_response = Mock()
mock_response.text = test_get_transfer_resp_xml
mock_response.status_code = 200
mock_response.history = [1]
mock_response_get = Mock()
mock_response_get.text = test_get_transfer_resp_xml
mock_response_get.status_code = 200
mock_response_get.history = [1]
mock_response_get.iter_content=Mock(return_value=[1])
mock_response_get_fail = Mock()
mock_response_get_fail.text = test_get_transfer_resp_xml
mock_response_get_fail.status_code = 503
mock_response_get_fail.history = [1]
mock_response_get_fail.raise_for_status = Mock(
side_effect=Exception('Failed.') )
# We mock the client session POST response to return desired
# XML with a list of endpoints. We then check that the post
# was called with the correct values.
# The "mock_requests_get" is called to actually get the data,
# and we ensure it is called with the correct endpoint. It also
# requires an "iter_content" method that returns an iterable
# to simulate streaming.
mock_session_post.return_value = mock_response
mock_requests_get.return_value = mock_response_get
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None, verify=False,
headers=test_headers)
mock_requests_get.assert_called_once_with(test_endpoint1, stream=True)
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# First GET (endpoint1) returns a failed response. The second
# GET (endpoint2) will work
mock_requests_get.side_effect = [mock_response_get_fail,
mock_response_get]
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None, verify=False,
headers=test_headers)
mock_requests_get.assert_has_calls(
[ call(test_endpoint1, stream=True),
call(test_endpoint2, stream=True) ] )
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Both endpoints, and therefore the transfer, fail
mock_requests_get.side_effect = [mock_response_get_fail,
mock_response_get_fail]
with self.assertRaises(TransferException):
c.transfer_file(test_localfile, uri=test_uri)
mock_session_post.assert_called_once_with(c.base_url,
data=test_get_transfer_xml,
json=None, verify=False,
headers=test_headers )
mock_requests_get.assert_has_calls(
[ call(test_endpoint1, stream=True),
call(test_endpoint2, stream=True) ] )
mock_open.reset_mock()
mock_requests_get.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
def test_put_file(self,mock_open,mock_requests_put,mock_session_post,
mock_isfile):
mock_file = MagicMock(spec=file)
mock_open.return_value = mock_file
c = DataClient(certfile=test_cert)
self.assertTrue(c.is_authorized)
#logger = c.get_logger(debug=True)
# a couple of mocked responses to requests
mock_response = Mock()
mock_response.text = test_put_transfer_resp_xml
mock_response.status_code = 200
mock_response.history = [1]
mock_response_fail = Mock()
mock_response_fail.text = test_put_transfer_resp_xml
mock_response_fail.status_code = 503
mock_response_fail.history = [1]
mock_response_fail.raise_for_status = Mock(
side_effect=Exception('Failed.') )
# make sure that transfer_file raises en error on bad inputs
with self.assertRaises(ValueError):
c.transfer_file(test_localfile)
# We mock the client session POST response to return desired
# XML with a list of endpoints. We then check that the post
# was called with the correct values.
# The "mock_requests_put" is called to actually put the data,
# and we ensure it is called with the correct endpoint.
#
# We do the whole thing three times: first time specifying a uri,
# second time specifying archive (filename for URI derived from
# local filename), and finally specifying both archive and filename.
for i in range(3):
mock_session_post.return_value = mock_response
mock_requests_put.return_value = mock_response
if i is 0:
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
elif i is 1:
c.transfer_file(test_localfile, archive=test_archive,
is_put=True)
else:
c.transfer_file(test_localfile, archive=test_archive,
filename=test_localfile, is_put=True)
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None, verify=False,
headers=test_headers)
mock_requests_put.assert_called_once_with(test_endpoint1,
data=mock_file)
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Test specifying the stream
mock_session_post.return_value = mock_response
mock_requests_put.return_value = mock_response
c.transfer_file(test_localfile, archive=test_archive,
filename=test_localfile, is_put=True,
stream=test_stream)
expected_headers = copy.deepcopy(test_headers)
expected_headers['X-CADC-Stream'] = test_stream
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None, verify=False,
headers=expected_headers)
mock_requests_put.assert_called_once_with(test_endpoint1,
data=mock_file)
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# First PUT (endpoint1) returns a failed response. The second
# PUT (endpoint2) will work
mock_requests_put.side_effect = [mock_response_fail, mock_response]
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None, verify=False,
headers=test_headers)
mock_requests_put.assert_has_calls(
[ call(test_endpoint1, data=mock_file),
call(test_endpoint2, data=mock_file) ] )
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# Both endpoints, and therefore the transfer, fail
mock_requests_put.side_effect = [mock_response_fail,
mock_response_fail]
with self.assertRaises(TransferException):
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
mock_session_post.assert_called_once_with(c.base_url,
data=test_put_transfer_xml,
json=None, verify=False,
headers=test_headers )
mock_requests_put.assert_has_calls(
[ call(test_endpoint1, data=mock_file),
call(test_endpoint2, data=mock_file) ] )
mock_open.reset_mock()
mock_requests_put.reset_mock()
mock_session_post.reset_mock()
mock_isfile.reset_mock()
# An anonymous put should raise an UnauthorizedException
c.is_authorized = False
with self.assertRaises(UnauthorizedException):
c.transfer_file(test_localfile, uri=test_uri, is_put=True)
