def test_constructor(self): # default client, patch netrc to ensure no netrc found with patch('netrc.netrc.authenticators') as mock_authenticator: mock_authenticator.side_effect = Exception('No netrc') c = DataClient() self.assertIsNone(c.certificate_file_location) self.assertIsNone(c.basic_auth) self.assertFalse(c.is_authorized) self.assertEqual(c.protocol, 'http') self.assertEqual(c.chunk_size, 1024) self.assertIsNotNone(c.transfer_reader) self.assertFalse(c.transfer_reader.validate) self.assertIsNotNone(c.transfer_writer) with patch('netrc.netrc.authenticators') as mock_authenticator: mock_authenticator.side_effect = Exception('No netrc') c = DataClient(schema_validate=True) self.assertTrue(c.transfer_reader.validate)
def test_data_info_netrc(self,mock_session_head, mock_authenticator): c = DataClient(certfile=test_cert, host=test_host) self.assertTrue(c.is_authorized) # a mocked response mock_response = Mock() mock_response.status_code = 200 mock_response.headers = {'key': 'value'} mock_response.history = [1] # Head request when we are using .netrc mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'http://%s/data/auth/%s/%s' % (test_host, test_archive, test_localfile), verify=False ) mock_session_head.reset_mock()
def test_data_info_netrc(self, mock_session_head, mock_authenticator): c = DataClient(certfile=test_cert, host=test_host) self.assertTrue(c.is_authorized) # a mocked response mock_response = Mock() mock_response.status_code = 200 mock_response.headers = {'key': 'value'} mock_response.history = [1] # Head request when we are using .netrc mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'http://%s/data/auth/%s/%s' % (test_host, test_archive, test_localfile), verify=False) mock_session_head.reset_mock()
def test_data_info_cert(self, mock_session_head, mock_isfile): c = DataClient(certfile=test_cert, host=test_host) self.assertTrue(c.is_authorized) # a mocked response mock_response = Mock() mock_response.status_code = 200 mock_response.headers = {'key': 'value'} mock_response.history = [1] # Head request when we have a cert mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'https://%s/data/pub/%s/%s' % (test_host, test_archive, test_localfile), verify=False) mock_session_head.reset_mock() mock_isfile.reset_mock() # We can also handle the anonymous case here c = DataClient(host=test_host, anonymous=True) self.assertFalse(c.is_authorized) mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'http://%s/data/pub/%s/%s' % (test_host, test_archive, test_localfile), verify=False) mock_session_head.reset_mock() mock_isfile.reset_mock()
def test_data_info_cert(self,mock_session_head, mock_isfile): c = DataClient(certfile=test_cert, host=test_host) self.assertTrue(c.is_authorized) # a mocked response mock_response = Mock() mock_response.status_code = 200 mock_response.headers = {'key': 'value'} mock_response.history = [1] # Head request when we have a cert mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'https://%s/data/pub/%s/%s' % (test_host, test_archive, test_localfile), verify=False ) mock_session_head.reset_mock() mock_isfile.reset_mock() # We can also handle the anonymous case here c = DataClient(host=test_host, anonymous=True) self.assertFalse(c.is_authorized) mock_session_head.return_value = mock_response c.data_info(test_archive, test_localfile) mock_session_head.assert_called_once_with( 'http://%s/data/pub/%s/%s' % (test_host, test_archive, test_localfile), verify=False ) mock_session_head.reset_mock() mock_isfile.reset_mock()
def test_get_file(self, mock_open, mock_requests_get, mock_session_post, mock_isfile): mock_file = MagicMock(spec=file) mock_open.return_value = mock_file c = DataClient(certfile=test_cert) self.assertTrue(c.is_authorized) #logger = c.get_logger(debug=True) # a couple of mocked responses to requests mock_response = Mock() mock_response.text = test_get_transfer_resp_xml mock_response.status_code = 200 mock_response.history = [1] mock_response_get = Mock() mock_response_get.text = test_get_transfer_resp_xml mock_response_get.status_code = 200 mock_response_get.history = [1] mock_response_get.iter_content = Mock(return_value=[1]) mock_response_get_fail = Mock() mock_response_get_fail.text = test_get_transfer_resp_xml mock_response_get_fail.status_code = 503 mock_response_get_fail.history = [1] mock_response_get_fail.raise_for_status = Mock( side_effect=Exception('Failed.')) # We mock the client session POST response to return desired # XML with a list of endpoints. We then check that the post # was called with the correct values. # The "mock_requests_get" is called to actually get the data, # and we ensure it is called with the correct endpoint. It also # requires an "iter_content" method that returns an iterable # to simulate streaming. mock_session_post.return_value = mock_response mock_requests_get.return_value = mock_response_get c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_get.assert_called_once_with(test_endpoint1, stream=True) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # First GET (endpoint1) returns a failed response. The second # GET (endpoint2) will work mock_requests_get.side_effect = [ mock_response_get_fail, mock_response_get ] c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_get.assert_has_calls([ call(test_endpoint1, stream=True), call(test_endpoint2, stream=True) ]) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Both endpoints, and therefore the transfer, fail mock_requests_get.side_effect = [ mock_response_get_fail, mock_response_get_fail ] with self.assertRaises(TransferException): c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_get.assert_has_calls([ call(test_endpoint1, stream=True), call(test_endpoint2, stream=True) ]) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock()
def test_put_file(self, mock_open, mock_requests_put, mock_session_post, mock_isfile): mock_file = MagicMock(spec=file) mock_open.return_value = mock_file c = DataClient(certfile=test_cert) self.assertTrue(c.is_authorized) #logger = c.get_logger(debug=True) # a couple of mocked responses to requests mock_response = Mock() mock_response.text = test_put_transfer_resp_xml mock_response.status_code = 200 mock_response.history = [1] mock_response_fail = Mock() mock_response_fail.text = test_put_transfer_resp_xml mock_response_fail.status_code = 503 mock_response_fail.history = [1] mock_response_fail.raise_for_status = Mock( side_effect=Exception('Failed.')) # make sure that transfer_file raises en error on bad inputs with self.assertRaises(ValueError): c.transfer_file(test_localfile) # We mock the client session POST response to return desired # XML with a list of endpoints. We then check that the post # was called with the correct values. # The "mock_requests_put" is called to actually put the data, # and we ensure it is called with the correct endpoint. # # We do the whole thing three times: first time specifying a uri, # second time specifying archive (filename for URI derived from # local filename), and finally specifying both archive and filename. for i in range(3): mock_session_post.return_value = mock_response mock_requests_put.return_value = mock_response if i is 0: c.transfer_file(test_localfile, uri=test_uri, is_put=True) elif i is 1: c.transfer_file(test_localfile, archive=test_archive, is_put=True) else: c.transfer_file(test_localfile, archive=test_archive, filename=test_localfile, is_put=True) mock_session_post.assert_called_once_with( c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_put.assert_called_once_with(test_endpoint1, data=mock_file) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Test specifying the stream mock_session_post.return_value = mock_response mock_requests_put.return_value = mock_response c.transfer_file(test_localfile, archive=test_archive, filename=test_localfile, is_put=True, stream=test_stream) expected_headers = copy.deepcopy(test_headers) expected_headers['X-CADC-Stream'] = test_stream mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=expected_headers) mock_requests_put.assert_called_once_with(test_endpoint1, data=mock_file) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # First PUT (endpoint1) returns a failed response. The second # PUT (endpoint2) will work mock_requests_put.side_effect = [mock_response_fail, mock_response] c.transfer_file(test_localfile, uri=test_uri, is_put=True) mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_put.assert_has_calls([ call(test_endpoint1, data=mock_file), call(test_endpoint2, data=mock_file) ]) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Both endpoints, and therefore the transfer, fail mock_requests_put.side_effect = [ mock_response_fail, mock_response_fail ] with self.assertRaises(TransferException): c.transfer_file(test_localfile, uri=test_uri, is_put=True) mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_put.assert_has_calls([ call(test_endpoint1, data=mock_file), call(test_endpoint2, data=mock_file) ]) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # An anonymous put should raise an UnauthorizedException c.is_authorized = False with self.assertRaises(UnauthorizedException): c.transfer_file(test_localfile, uri=test_uri, is_put=True)
def test_get_file(self,mock_open,mock_requests_get,mock_session_post, mock_isfile): mock_file = MagicMock(spec=file) mock_open.return_value = mock_file c = DataClient(certfile=test_cert) self.assertTrue(c.is_authorized) #logger = c.get_logger(debug=True) # a couple of mocked responses to requests mock_response = Mock() mock_response.text = test_get_transfer_resp_xml mock_response.status_code = 200 mock_response.history = [1] mock_response_get = Mock() mock_response_get.text = test_get_transfer_resp_xml mock_response_get.status_code = 200 mock_response_get.history = [1] mock_response_get.iter_content=Mock(return_value=[1]) mock_response_get_fail = Mock() mock_response_get_fail.text = test_get_transfer_resp_xml mock_response_get_fail.status_code = 503 mock_response_get_fail.history = [1] mock_response_get_fail.raise_for_status = Mock( side_effect=Exception('Failed.') ) # We mock the client session POST response to return desired # XML with a list of endpoints. We then check that the post # was called with the correct values. # The "mock_requests_get" is called to actually get the data, # and we ensure it is called with the correct endpoint. It also # requires an "iter_content" method that returns an iterable # to simulate streaming. mock_session_post.return_value = mock_response mock_requests_get.return_value = mock_response_get c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_get.assert_called_once_with(test_endpoint1, stream=True) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # First GET (endpoint1) returns a failed response. The second # GET (endpoint2) will work mock_requests_get.side_effect = [mock_response_get_fail, mock_response_get] c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_get.assert_has_calls( [ call(test_endpoint1, stream=True), call(test_endpoint2, stream=True) ] ) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Both endpoints, and therefore the transfer, fail mock_requests_get.side_effect = [mock_response_get_fail, mock_response_get_fail] with self.assertRaises(TransferException): c.transfer_file(test_localfile, uri=test_uri) mock_session_post.assert_called_once_with(c.base_url, data=test_get_transfer_xml, json=None, verify=False, headers=test_headers ) mock_requests_get.assert_has_calls( [ call(test_endpoint1, stream=True), call(test_endpoint2, stream=True) ] ) mock_open.reset_mock() mock_requests_get.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock()
def test_put_file(self,mock_open,mock_requests_put,mock_session_post, mock_isfile): mock_file = MagicMock(spec=file) mock_open.return_value = mock_file c = DataClient(certfile=test_cert) self.assertTrue(c.is_authorized) #logger = c.get_logger(debug=True) # a couple of mocked responses to requests mock_response = Mock() mock_response.text = test_put_transfer_resp_xml mock_response.status_code = 200 mock_response.history = [1] mock_response_fail = Mock() mock_response_fail.text = test_put_transfer_resp_xml mock_response_fail.status_code = 503 mock_response_fail.history = [1] mock_response_fail.raise_for_status = Mock( side_effect=Exception('Failed.') ) # make sure that transfer_file raises en error on bad inputs with self.assertRaises(ValueError): c.transfer_file(test_localfile) # We mock the client session POST response to return desired # XML with a list of endpoints. We then check that the post # was called with the correct values. # The "mock_requests_put" is called to actually put the data, # and we ensure it is called with the correct endpoint. # # We do the whole thing three times: first time specifying a uri, # second time specifying archive (filename for URI derived from # local filename), and finally specifying both archive and filename. for i in range(3): mock_session_post.return_value = mock_response mock_requests_put.return_value = mock_response if i is 0: c.transfer_file(test_localfile, uri=test_uri, is_put=True) elif i is 1: c.transfer_file(test_localfile, archive=test_archive, is_put=True) else: c.transfer_file(test_localfile, archive=test_archive, filename=test_localfile, is_put=True) mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_put.assert_called_once_with(test_endpoint1, data=mock_file) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Test specifying the stream mock_session_post.return_value = mock_response mock_requests_put.return_value = mock_response c.transfer_file(test_localfile, archive=test_archive, filename=test_localfile, is_put=True, stream=test_stream) expected_headers = copy.deepcopy(test_headers) expected_headers['X-CADC-Stream'] = test_stream mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=expected_headers) mock_requests_put.assert_called_once_with(test_endpoint1, data=mock_file) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # First PUT (endpoint1) returns a failed response. The second # PUT (endpoint2) will work mock_requests_put.side_effect = [mock_response_fail, mock_response] c.transfer_file(test_localfile, uri=test_uri, is_put=True) mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers) mock_requests_put.assert_has_calls( [ call(test_endpoint1, data=mock_file), call(test_endpoint2, data=mock_file) ] ) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # Both endpoints, and therefore the transfer, fail mock_requests_put.side_effect = [mock_response_fail, mock_response_fail] with self.assertRaises(TransferException): c.transfer_file(test_localfile, uri=test_uri, is_put=True) mock_session_post.assert_called_once_with(c.base_url, data=test_put_transfer_xml, json=None, verify=False, headers=test_headers ) mock_requests_put.assert_has_calls( [ call(test_endpoint1, data=mock_file), call(test_endpoint2, data=mock_file) ] ) mock_open.reset_mock() mock_requests_put.reset_mock() mock_session_post.reset_mock() mock_isfile.reset_mock() # An anonymous put should raise an UnauthorizedException c.is_authorized = False with self.assertRaises(UnauthorizedException): c.transfer_file(test_localfile, uri=test_uri, is_put=True)