def _get_rest_client( endpoint, cert=None, key=None, pem=None, ca=None, # pylint: disable=invalid-name, too-many-arguments aad=False, no_verify=False): """ Get the rest client to send a http request with secured connections :param endpoint: See select command in this file :param cert: See select command in this file :param key: See select command in this file :param pem: See select command in this file :param ca: See select command in this file :param aad: See select command in this file :param no_verify: See select command in this file :return: ServiceClient from msrest """ if aad: new_token, new_cache = get_aad_token(endpoint, no_verify) set_aad_cache(new_token, new_cache) return ServiceClient(AdalAuthentication(no_verify), Configuration(endpoint)) # If the code reaches here, it is not AAD return ServiceClient(_get_client_cert_auth(pem, cert, key, ca, no_verify), Configuration(endpoint))
def select(endpoint, cert=None, key=None, pem=None, ca=None, aad=False, no_verify=False): #pylint: disable-msg=too-many-locals """ Connects to a Service Fabric cluster endpoint. If connecting to secure cluster specify an absolute path to a cert (.crt) and key file (.key) or a single file with both (.pem). Do not specify both. Optionally, if connecting to a secure cluster, specify also an absolute path to a CA bundle file or directory of trusted CA certs. :param str endpoint: Cluster endpoint URL, including port and HTTP or HTTPS prefix :param str cert: Absolute path to a client certificate file :param str key: Absolute path to client certificate key file :param str pem: Absolute path to client certificate, as a .pem file :param str ca: Absolute path to CA certs directory to treat as valid or CA bundle file :param bool aad: Use Azure Active Directory for authentication :param bool no_verify: Disable verification for certificates when using HTTPS, note: this is an insecure option and should not be used for production environments """ from sfctl.config import (set_ca_cert, set_auth, set_aad_cache, set_cluster_endpoint, set_no_verify) from msrest import ServiceClient, Configuration from sfctl.auth import ClientCertAuthentication, AdalAuthentication select_arg_verify(endpoint, cert, key, pem, ca, aad, no_verify) if aad: new_token, new_cache = get_aad_token(endpoint, no_verify) set_aad_cache(new_token, new_cache) rest_client = ServiceClient(AdalAuthentication(no_verify), Configuration(endpoint)) # Make sure basic GET request succeeds rest_client.send(rest_client.get('/')).raise_for_status() else: client_cert = None if pem: client_cert = pem elif cert: client_cert = (cert, key) rest_client = ServiceClient( ClientCertAuthentication(client_cert, ca, no_verify), Configuration(endpoint)) # Make sure basic GET request succeeds rest_client.send(rest_client.get('/')).raise_for_status() set_cluster_endpoint(endpoint) set_no_verify(no_verify) set_ca_cert(ca) set_auth(pem, cert, key, aad)
def test_request_proxy(self): cfg = Configuration("http://my_service.com") cfg.proxies.add("http://my_service.com", 'http://localhost:57979') creds = Authentication() def hook(adptr, request, *args, **kwargs): self.assertEqual(kwargs.get('proxies'), {"http://my_service.com":'http://localhost:57979'}) return kwargs['result'] client = ServiceClient(creds, cfg) client.add_hook('request', hook, precall=False, overwrite=True) request = client.get("/get_endpoint", {'check':True}) response = client.send(request) os.environ['HTTPS_PROXY'] = "http://localhost:1987" def hook2(adptr, request, *args, **kwargs): self.assertEqual(kwargs.get('proxies')['https'], "http://localhost:1987") return kwargs['result'] cfg = Configuration("http://my_service.com") client = ServiceClient(creds, cfg) client.add_hook('request', hook2, precall=False, overwrite=True) request = client.get("/get_endpoint", {'check':True}) response = client.send(request) del os.environ['HTTPS_PROXY']
def test_request_proxy(self): # Note that this test requires requests >= 2.8.0 to accept host on proxy cfg = Configuration("http://my_service.com") cfg.proxies.add("http://my_service.com", 'http://localhost:57979') cfg.credentials = Authentication() httpretty.register_uri(httpretty.GET, "http://localhost:57979/get_endpoint?check=True", body='"Mocked body"', content_type="application/json", status=200) client = ServiceClient(None, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) assert response.json() == "Mocked body" with mock.patch.dict('os.environ', {'HTTP_PROXY': "http://localhost:1987"}): httpretty.register_uri(httpretty.GET, "http://localhost:1987/get_endpoint?check=True", body='"Mocked body"', content_type="application/json", status=200) cfg = Configuration("http://my_service.com") client = ServiceClient(None, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) assert response.json() == "Mocked body"
def setUp(self): cfg = Configuration("https://my_service.com") cfg.retry_policy.backoff_factor = 0 creds = Authentication() self.client = ServiceClient(creds, cfg) self.request = self.client.get("/get_endpoint", {'check': True}) return super(TestRuntimeRetry, self).setUp()
def setUp(self): cfg = Configuration("https://my_service.com") cfg.retry_policy.backoff_factor=0 cfg.redirect_policy.max_redirects=2 creds = Authentication() self.client = ServiceClient(creds, cfg) return super(TestRedirect, self).setUp()
def test_credential_headers(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", body='[{"title": "Test Data"}]', content_type="application/json") token = { 'access_token': 'eswfld123kjhn1v5423', 'refresh_token': 'asdfkljh23490sdf', 'token_type': 'Bearer', 'expires_in': '3600', } creds = OAuthTokenAuthentication("client_id", token) cfg = Configuration("https://my_service.com") client = ServiceClient(creds, cfg) def hook(aptr, req, *args, **kwargs): self.assertTrue('Authorization' in req.headers) self.assertEqual(req.headers['Authorization'], 'Bearer eswfld123kjhn1v5423') client.add_hook('request', hook) request = client.get("/get_endpoint", {'check':True}) response = client.send(request) check = httpretty.last_request() self.assertEqual(response.json(), [{"title": "Test Data"}]) token['expires_in'] = '-30' creds = OAuthTokenAuthentication("client_id", token) client = ServiceClient(creds, cfg) request = client.get("/get_endpoint", {'check':True}) with self.assertRaises(TokenExpiredError): response = client.send(request)
def test_session_callback(self): client = ServiceClient(self.creds, self.cfg) local_session = requests.Session() def callback(session, global_config, local_config, **kwargs): self.assertIs(session, local_session) self.assertIs(global_config, self.cfg) self.assertTrue(local_config["test"]) return {'used_callback': True} self.cfg.session_configuration_callback = callback output_kwargs = client._configure_session(local_session, **{"test": True}) self.assertTrue(output_kwargs['used_callback'])
def test_format_data(self): mock_client = mock.create_autospec(ServiceClient) data = ServiceClient._format_data(mock_client, None) self.assertEqual(data, (None, None)) data = ServiceClient._format_data(mock_client, "Test") self.assertEqual(data, (None, "Test")) mock_stream = mock.create_autospec(io.BytesIO) data = ServiceClient._format_data(mock_client, mock_stream) self.assertEqual(data, (None, mock_stream, "application/octet-stream")) mock_stream.name = "file_name" data = ServiceClient._format_data(mock_client, mock_stream) self.assertEqual(data, ("file_name", mock_stream, "application/octet-stream"))
def __init__(self, credentials, subscription_id, base_url=None): self.config = LogAnalyticsManagementClientConfiguration( credentials, subscription_id, base_url) self._client = ServiceClient(self.config.credentials, self.config) client_models = { k: v for k, v in models.__dict__.items() if isinstance(v, type) } self._serialize = Serializer(client_models) self._serialize.basic_types[unicode] = "unicode" self._serialize.serialize_type["dict"] = self._patched_serialize_dict self._deserialize = Deserializer(client_models) self._deserialize.basic_types[unicode] = "unicode" self._deserialize.deserialize_type[ "dict"] = self._patched_deserialize_dict self._deserialize.deserialize_type[ "list"] = self._patched_deserialize_iter self.alert_services = AlertServicesOperations(self._client, self.config, self._serialize, self._deserialize)
def test_request_builder(self): client = ServiceClient(self.creds, self.cfg) req = client.get('http://127.0.0.1/') assert req.method == 'GET' assert req.url == 'http://127.0.0.1/' assert req.headers == {'Accept': 'application/json'} assert req.data is None assert req.files is None req = client.put("http://127.0.0.1/", content={'creation': True}) assert req.method == 'PUT' assert req.url == "http://127.0.0.1/" assert req.headers == {'Content-Length': '18', 'Accept': 'application/json'} assert req.data == '{"creation": true}' assert req.files is None
def test_format_url(self): url = "/bool/test true" mock_client = mock.create_autospec(ServiceClient) mock_client.config = mock.Mock(base_url = "http://localhost:3000") formatted = ServiceClient._format_url(mock_client, url) self.assertEqual(formatted, "http://localhost:3000/bool/test true") url = "https://absolute_url.com/my/test/path" formatted = ServiceClient._format_url(mock_client, url) self.assertEqual(formatted, "https://absolute_url.com/my/test/path") url = "test" formatted = ServiceClient._format_url(mock_client, url) self.assertEqual(formatted, "http://localhost:3000/test")
def setUp(self): cfg = Configuration("https://my_service.com") cfg.retry_policy.backoff_factor=0 creds = Authentication() self.client = ServiceClient(creds, cfg) self.request = self.client.get("/get_endpoint", {'check':True}) return super(TestRuntimeRetry, self).setUp()
def test_client_send(self): mock_client = mock.create_autospec(ServiceClient) mock_client.config = self.cfg mock_client.creds = self.creds mock_client._log = self.cfg._log mock_client._configure_session.return_value = {} session = mock.create_autospec(requests.Session) mock_client.creds.signed_session.return_value = session mock_client.creds.refresh_session.return_value = session request = ClientRequest('GET') ServiceClient.send(mock_client, request) session.request.call_count = 0 mock_client._configure_session.assert_called_with(session) session.request.assert_called_with('GET', None, data=[], headers={}) session.close.assert_called_with() ServiceClient.send(mock_client, request, headers={'id':'1234'}, content={'Test':'Data'}) mock_client._configure_session.assert_called_with(session) session.request.assert_called_with('GET', None, data='{"Test": "Data"}', headers={'Content-Length': 16, 'id':'1234'}) self.assertEqual(session.request.call_count, 1) session.request.call_count = 0 session.close.assert_called_with() session.request.side_effect = requests.RequestException("test") with self.assertRaises(ClientRequestError): ServiceClient.send(mock_client, request, headers={'id':'1234'}, content={'Test':'Data'}, test='value') mock_client._configure_session.assert_called_with(session, test='value') session.request.assert_called_with('GET', None, data='{"Test": "Data"}', headers={'Content-Length': 16, 'id':'1234'}) self.assertEqual(session.request.call_count, 1) session.request.call_count = 0 session.close.assert_called_with() session.request.side_effect = oauth2.rfc6749.errors.InvalidGrantError("test") with self.assertRaises(TokenExpiredError): ServiceClient.send(mock_client, request, headers={'id':'1234'}, content={'Test':'Data'}, test='value') self.assertEqual(session.request.call_count, 2) session.request.call_count = 0 session.close.assert_called_with() session.request.side_effect = ValueError("test") with self.assertRaises(ValueError): ServiceClient.send(mock_client, request, headers={'id':'1234'}, content={'Test':'Data'}, test='value') session.close.assert_called_with()
def test_format_url(self): url = "/bool/test true" mock_client = mock.create_autospec(ServiceClient) mock_client.config = mock.Mock(base_url="http://localhost:3000") formatted = ServiceClient.format_url(mock_client, url) self.assertEqual(formatted, "http://localhost:3000/bool/test true") mock_client.config = mock.Mock(base_url="http://localhost:3000/") formatted = ServiceClient.format_url(mock_client, url, foo=123, bar="value") self.assertEqual(formatted, "http://localhost:3000/bool/test true") url = "https://absolute_url.com/my/test/path" formatted = ServiceClient.format_url(mock_client, url) self.assertEqual(formatted, "https://absolute_url.com/my/test/path") formatted = ServiceClient.format_url(mock_client, url, foo=123, bar="value") self.assertEqual(formatted, "https://absolute_url.com/my/test/path") url = "test" formatted = ServiceClient.format_url(mock_client, url) self.assertEqual(formatted, "http://localhost:3000/test") mock_client.config = mock.Mock(base_url="http://{hostname}:{port}/{foo}/{bar}") formatted = ServiceClient.format_url(mock_client, url, hostname="localhost", port="3000", foo=123, bar="value") self.assertEqual(formatted, "http://localhost:3000/123/value/test") mock_client.config = mock.Mock(base_url="https://my_endpoint.com/") formatted = ServiceClient.format_url(mock_client, url, foo=123, bar="value") self.assertEqual(formatted, "https://my_endpoint.com/test")
def test_format_url(self): url = "/bool/test true" client = mock.create_autospec(ServiceClient) client.config = mock.Mock(base_url="http://localhost:3000") formatted = ServiceClient.format_url(client, url) self.assertEqual(formatted, "http://localhost:3000/bool/test true") client.config = mock.Mock(base_url="http://localhost:3000/") formatted = ServiceClient.format_url(client, url, foo=123, bar="value") self.assertEqual(formatted, "http://localhost:3000/bool/test true") url = "https://absolute_url.com/my/test/path" formatted = ServiceClient.format_url(client, url) self.assertEqual(formatted, "https://absolute_url.com/my/test/path") formatted = ServiceClient.format_url(client, url, foo=123, bar="value") self.assertEqual(formatted, "https://absolute_url.com/my/test/path") url = "test" formatted = ServiceClient.format_url(client, url) self.assertEqual(formatted, "http://localhost:3000/test") client.config = mock.Mock(base_url="http://{hostname}:{port}/{foo}/{bar}") formatted = ServiceClient.format_url(client, url, hostname="localhost", port="3000", foo=123, bar="value") self.assertEqual(formatted, "http://localhost:3000/123/value/test") client.config = mock.Mock(base_url="https://my_endpoint.com/") formatted = ServiceClient.format_url(client, url, foo=123, bar="value") self.assertEqual(formatted, "https://my_endpoint.com/test")
def test_request_fail(self, mock_requests): mock_requests.return_value.request.return_value = "test" cfg = Configuration("https://my_service.com") creds = Authentication() client = ServiceClient(creds, cfg) request = client.get("/get_endpoint", {'check':True}) response = client.send(request) check = httpretty.last_request() self.assertEqual(response, "test") mock_requests.return_value.request.side_effect = requests.RequestException with self.assertRaises(ClientRequestError): client.send(request)
def test_request_builder(self): client = ServiceClient(self.creds, self.cfg) req = client.get('http://example.org') assert req.method == 'GET' assert req.url == 'http://example.org' assert req.params == {} assert req.headers == {'Accept': 'application/json'} assert req.data == [] assert req.files == [] req = client.put('http://example.org', content={'creation': True}) assert req.method == 'PUT' assert req.url == 'http://example.org' assert req.params == {} assert req.headers == { 'Content-Length': '18', 'Accept': 'application/json' } assert req.data == '{"creation": true}' assert req.files == []
def _credential_setup(self): auth_options = dict( auth_source=self.get_option('auth_source'), profile=self.get_option('profile'), subscription_id=self.get_option('subscription_id'), client_id=self.get_option('client_id'), secret=self.get_option('secret'), tenant=self.get_option('tenant'), ad_user=self.get_option('ad_user'), password=self.get_option('password'), cloud_environment=self.get_option('cloud_environment'), cert_validation_mode=self.get_option('cert_validation_mode'), api_profile=self.get_option('api_profile'), adfs_authority_url=self.get_option('adfs_authority_url') ) self.azure_auth = AzureRMAuth(**auth_options) self._clientconfig = AzureRMRestConfiguration(self.azure_auth.azure_credentials, self.azure_auth.subscription_id, self.azure_auth._cloud_environment.endpoints.resource_manager) self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig)
def test_client_formdata_send(self): mock_client = mock.create_autospec(ServiceClient) mock_client._format_data.return_value = "formatted" request = ClientRequest('GET') ServiceClient.send_formdata(mock_client, request) mock_client.send.assert_called_with(request, {}, None, files={}) ServiceClient.send_formdata(mock_client, request, {'id': '1234'}, {'Test': 'Data'}) mock_client.send.assert_called_with(request, {'id': '1234'}, None, files={'Test': 'formatted'}) ServiceClient.send_formdata(mock_client, request, {'Content-Type': '1234'}, { '1': '1', '2': '2' }) mock_client.send.assert_called_with(request, {}, None, files={ '1': 'formatted', '2': 'formatted' })
def test_client_add_hook(self): client = ServiceClient(self.creds, self.cfg) def hook(): pass client.add_hook("request", hook) self.assertTrue(hook in client._adapter._client_hooks['request'].precalls) client.add_hook("request", hook, precall=False) self.assertTrue(hook in client._adapter._client_hooks['request'].postcalls) client.remove_hook("request", hook) self.assertFalse(hook in client._adapter._client_hooks['request'].precalls) self.assertFalse(hook in client._adapter._client_hooks['request'].postcalls)
def test_credential_headers(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", body='[{"title": "Test Data"}]', content_type="application/json") token = { 'access_token': 'eswfld123kjhn1v5423', 'refresh_token': 'asdfkljh23490sdf', 'token_type': 'Bearer', 'expires_in': '3600', } creds = OAuthTokenAuthentication("client_id", token) cfg = Configuration("https://my_service.com") client = ServiceClient(creds, cfg) def hook(aptr, req, *args, **kwargs): self.assertTrue('Authorization' in req.headers) self.assertEqual(req.headers['Authorization'], 'Bearer eswfld123kjhn1v5423') client.add_hook('request', hook) request = client.get("/get_endpoint", {'check': True}) response = client.send(request) check = httpretty.last_request() self.assertEqual(response.json(), [{"title": "Test Data"}]) token['expires_in'] = '-30' creds = OAuthTokenAuthentication("client_id", token) client = ServiceClient(creds, cfg) request = client.get("/get_endpoint", {'check': True}) with self.assertRaises(TokenExpiredError): response = client.send(request)
def test_credential_headers(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", body='[{"title": "Test Data"}]', content_type="application/json") token = { 'access_token': 'eswfld123kjhn1v5423', 'refresh_token': 'asdfkljh23490sdf', 'token_type': 'Bearer', 'expires_in': '3600', } cfg = Configuration("https://my_service.com") cfg.credentials = OAuthTokenAuthentication("client_id", token) client = ServiceClient(None, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) assert 'Authorization' in response.request.headers assert response.request.headers['Authorization'] == 'Bearer eswfld123kjhn1v5423' httpretty.has_request() assert response.json() == [{"title": "Test Data"}] # Expiration test token['expires_in'] = '-30' cfg.credentials = OAuthTokenAuthentication("client_id", token) client = ServiceClient(None, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) with pytest.raises(TokenExpiredError): response = client.send(request)
def test_keep_alive(self): cfg = Configuration("http://127.0.0.1/") cfg.keep_alive = True class Creds(Authentication): def __init__(self): self.first_session = None self.called = 0 def signed_session(self, session=None): self.called += 1 assert session is not None if self.first_session: assert self.first_session is session else: self.first_session = session creds = Creds() client = ServiceClient(creds, cfg) req = client.get() try: client.send( req ) # Will fail, I don't care, that's not the point of the test except Exception: pass try: client.send( req ) # Will fail, I don't care, that's not the point of the test except Exception: pass assert creds.called == 2 assert client._http_driver.session # Still alive # Manually close the client in "keep_alive" mode client.close()
def test_client_formdata_send(self): mock_client = mock.create_autospec(ServiceClient) mock_client._format_data.return_value = "formatted" request = ClientRequest('GET') ServiceClient.send_formdata(mock_client, request) mock_client.send.assert_called_with(request, {}, None, files={}) ServiceClient.send_formdata(mock_client, request, {'id':'1234'}, {'Test':'Data'}) mock_client.send.assert_called_with(request, {'id':'1234'}, None, files={'Test':'formatted'}) ServiceClient.send_formdata(mock_client, request, {'Content-Type':'1234'}, {'1':'1', '2':'2'}) mock_client.send.assert_called_with(request, {}, None, files={'1':'formatted', '2':'formatted'})
def test_request_fail(self, mock_requests): mock_requests.return_value.request.return_value = mock.Mock(text="text") cfg = Configuration("https://my_service.com") cfg.credentials = Authentication() client = ServiceClient(None, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) assert response.text == "text" mock_requests.return_value.request.side_effect = requests.RequestException with self.assertRaises(ClientRequestError): client.send(request)
def test_request_fail(self, mock_requests): mock_requests.return_value.request.return_value = mock.Mock(_content_consumed=True) cfg = Configuration("https://my_service.com") creds = Authentication() client = ServiceClient(creds, cfg) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) check = httpretty.last_request() self.assertTrue(response._content_consumed) mock_requests.return_value.request.side_effect = requests.RequestException with self.assertRaises(ClientRequestError): client.send(request)
def test_context_manager(self): cfg = Configuration("http://127.0.0.1/") class Creds(Authentication): def __init__(self): self.first_session = None self.called = 0 def signed_session(self, session=None): self.called += 1 assert session is not None if self.first_session: assert self.first_session is session else: self.first_session = session creds = Creds() with ServiceClient(creds, cfg) as client: assert cfg.keep_alive req = client.get() try: client.send( req ) # Will fail, I don't care, that's not the point of the test except Exception: pass try: client.send( req ) # Will fail, I don't care, that's not the point of the test except Exception: pass assert client._http_driver.session # Still alive assert not cfg.keep_alive assert creds.called == 2
def test_request_fail(self, mock_requests): mock_requests.return_value.request.return_value = "test" cfg = Configuration("https://my_service.com") creds = Authentication() client = ServiceClient(creds, cfg) request = client.get("/get_endpoint", {'check': True}) response = client.send(request) check = httpretty.last_request() self.assertEqual(response, "test") mock_requests.return_value.request.side_effect = requests.RequestException with self.assertRaises(ClientRequestError): client.send(request)
class InventoryModule(BaseInventoryPlugin, Constructable): NAME = 'azure_rm' def __init__(self): super(InventoryModule, self).__init__() self._serializer = Serializer() self._deserializer = Deserializer() self._hosts = [] self._filters = None # FUTURE: use API profiles with defaults self._compute_api_version = '2017-03-30' self._network_api_version = '2015-06-15' self._default_header_parameters = { 'Content-Type': 'application/json; charset=utf-8' } self._request_queue = Queue() self.azure_auth = None self._batch_fetch = False def verify_file(self, path): ''' :param loader: an ansible.parsing.dataloader.DataLoader object :param path: the path to the inventory config file :return the contents of the config file ''' if super(InventoryModule, self).verify_file(path): if re.match(r'.{0,}azure_rm\.y(a)?ml$', path): return True # display.debug("azure_rm inventory filename must end with 'azure_rm.yml' or 'azure_rm.yaml'") return False def parse(self, inventory, loader, path, cache=True): super(InventoryModule, self).parse(inventory, loader, path) self._read_config_data(path) if self.get_option('use_contrib_script_compatible_sanitization'): self._sanitize_group_name = self._legacy_script_compatible_group_sanitization self._batch_fetch = self.get_option('batch_fetch') self._legacy_hostnames = self.get_option('plain_host_names') self._filters = self.get_option( 'exclude_host_filters') + self.get_option('default_host_filters') try: self._credential_setup() self._get_hosts() except Exception: raise def _credential_setup(self): auth_options = dict( auth_source=self.get_option('auth_source'), profile=self.get_option('profile'), subscription_id=self.get_option('subscription_id'), client_id=self.get_option('client_id'), secret=self.get_option('secret'), tenant=self.get_option('tenant'), ad_user=self.get_option('ad_user'), password=self.get_option('password'), cloud_environment=self.get_option('cloud_environment'), cert_validation_mode=self.get_option('cert_validation_mode'), api_profile=self.get_option('api_profile'), adfs_authority_url=self.get_option('adfs_authority_url')) self.azure_auth = AzureRMAuth(**auth_options) self._clientconfig = AzureRMRestConfiguration( self.azure_auth.azure_credentials, self.azure_auth.subscription_id, self.azure_auth._cloud_environment.endpoints.resource_manager) self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig) def _enqueue_get(self, url, api_version, handler, handler_args=None): if not handler_args: handler_args = {} self._request_queue.put_nowait( UrlAction(url=url, api_version=api_version, handler=handler, handler_args=handler_args)) def _enqueue_vm_list(self, rg='*'): if not rg or rg == '*': url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines' else: url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines' url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response) def _enqueue_vmss_list(self, rg=None): if not rg or rg == '*': url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets' else: url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets' url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vmss_page_response) def _get_hosts(self): for vm_rg in self.get_option('include_vm_resource_groups'): self._enqueue_vm_list(vm_rg) for vmss_rg in self.get_option('include_vmss_resource_groups'): self._enqueue_vmss_list(vmss_rg) if self._batch_fetch: self._process_queue_batch() else: self._process_queue_serial() constructable_config_strict = boolean( self.get_option('fail_on_template_errors')) constructable_config_compose = self.get_option('hostvar_expressions') constructable_config_groups = self.get_option('conditional_groups') constructable_config_keyed_groups = self.get_option('keyed_groups') for h in self._hosts: inventory_hostname = self._get_hostname(h) if self._filter_host(inventory_hostname, h.hostvars): continue self.inventory.add_host(inventory_hostname) # FUTURE: configurable default IP list? can already do this via hostvar_expressions self.inventory.set_variable( inventory_hostname, "ansible_host", next( chain(h.hostvars['public_ipv4_addresses'], h.hostvars['private_ipv4_addresses']), None)) for k, v in iteritems(h.hostvars): # FUTURE: configurable hostvar prefix? Makes docs harder... self.inventory.set_variable(inventory_hostname, k, v) # constructable delegation self._set_composite_vars(constructable_config_compose, h.hostvars, inventory_hostname, strict=constructable_config_strict) self._add_host_to_composed_groups( constructable_config_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) self._add_host_to_keyed_groups(constructable_config_keyed_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) # FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host def _filter_host(self, inventory_hostname, hostvars): self.templar.available_variables = hostvars for condition in self._filters: # FUTURE: should warn/fail if conditional doesn't return True or False conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format( condition) try: if boolean(self.templar.template(conditional)): return True except Exception as e: if boolean(self.get_option('fail_on_template_errors')): raise AnsibleParserError( "Error evaluating filter condition '{0}' for host {1}: {2}" .format(condition, inventory_hostname, to_native(e))) continue return False def _get_hostname(self, host): # FUTURE: configurable hostname sources return host.default_inventory_hostname def _process_queue_serial(self): try: while True: item = self._request_queue.get_nowait() resp = self.send_request(item.url, item.api_version) item.handler(resp, **item.handler_args) except Empty: pass def _on_vm_page_response(self, response, vmss=None): next_link = response.get('nextLink') if next_link: self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vm_page_response) if 'value' in response: for h in response['value']: # FUTURE: add direct VM filtering by tag here (performance optimization)? self._hosts.append( AzureHost(h, self, vmss=vmss, legacy_name=self._legacy_hostnames)) def _on_vmss_page_response(self, response): next_link = response.get('nextLink') if next_link: self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vmss_page_response) # FUTURE: add direct VMSS filtering by tag here (performance optimization)? for vmss in response['value']: url = '{0}/virtualMachines'.format(vmss['id']) # VMSS instances look close enough to regular VMs that we can share the handler impl... self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response, handler_args=dict(vmss=vmss)) # use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip # def _process_queue_batch(self): while True: batch_requests = [] batch_item_index = 0 batch_response_handlers = dict() try: while batch_item_index < 100: item = self._request_queue.get_nowait() name = str(uuid.uuid4()) query_parameters = {'api-version': item.api_version} req = self._client.get(item.url, query_parameters) batch_requests.append( dict(httpMethod="GET", url=req.url, name=name)) batch_response_handlers[name] = item batch_item_index += 1 except Empty: pass if not batch_requests: break batch_resp = self._send_batch(batch_requests) key_name = None if 'responses' in batch_resp: key_name = 'responses' elif 'value' in batch_resp: key_name = 'value' else: raise AnsibleError( "didn't find expected key responses/value in batch response" ) for idx, r in enumerate(batch_resp[key_name]): status_code = r.get('httpStatusCode') returned_name = r['name'] result = batch_response_handlers[returned_name] if status_code != 200: # FUTURE: error-tolerant operation mode (eg, permissions) raise AnsibleError( "a batched request failed with status code {0}, url {1}" .format(status_code, result.url)) # FUTURE: store/handle errors from individual handlers result.handler(r['content'], **result.handler_args) def _send_batch(self, batched_requests): url = '/batch' query_parameters = {'api-version': '2015-11-01'} body_obj = dict(requests=batched_requests) body_content = self._serializer.body(body_obj, 'object') header = {'x-ms-client-request-id': str(uuid.uuid4())} header.update(self._default_header_parameters) request = self._client.post(url, query_parameters) initial_response = self._client.send(request, header, body_content) # FUTURE: configurable timeout? poller = ARMPolling(timeout=2) poller.initialize( client=self._client, initial_response=initial_response, deserialization_callback=lambda r: self._deserializer('object', r)) poller.run() return poller.resource() def send_request(self, url, api_version): query_parameters = {'api-version': api_version} req = self._client.get(url, query_parameters) resp = self._client.send(req, self._default_header_parameters, stream=False) resp.raise_for_status() content = resp.content return json.loads(content) @staticmethod def _legacy_script_compatible_group_sanitization(name): # note that while this mirrors what the script used to do, it has many issues with unicode and usability in python regex = re.compile(r"[^A-Za-z0-9\_\-]") return regex.sub('_', name)
def test_client_header(self): client = ServiceClient(self.creds, self.cfg) client.add_header("test", "value") self.assertEqual(client._headers.get('test'), 'value')
def test_client_request(self): client = ServiceClient(self.creds, self.cfg) obj = client.get() self.assertEqual(obj.method, 'GET') self.assertIsNone(obj.url) self.assertEqual(obj.params, {}) obj = client.get("/service", {'param': "testing"}) self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service?param=testing") self.assertEqual(obj.params, {}) obj = client.get("service 2") self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service 2") self.cfg.base_url = "https://my_endpoint.com/" obj = client.get("//service3") self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service3") obj = client.put() self.assertEqual(obj.method, 'PUT') obj = client.post() self.assertEqual(obj.method, 'POST') obj = client.head() self.assertEqual(obj.method, 'HEAD') obj = client.merge() self.assertEqual(obj.method, 'MERGE') obj = client.patch() self.assertEqual(obj.method, 'PATCH') obj = client.delete() self.assertEqual(obj.method, 'DELETE')
def select( endpoint, cert=None, key=None, pem=None, ca=None, #pylint: disable=invalid-name, too-many-arguments aad=False, no_verify=False): #pylint: disable-msg=too-many-locals """ Connects to a Service Fabric cluster endpoint. If connecting to secure cluster, specify an absolute path to a cert (.crt) and key file (.key) or a single file with both (.pem). Do not specify both. Optionally, if connecting to a secure cluster, also specify an absolute path to a CA bundle file or directory of trusted CA certs. If using a directory of CA certs, `c_rehash <directory>` provided by OpenSSL must be run first to compute the certificate hashes and create the appropriate symbolics links. :param str endpoint: Cluster endpoint URL, including port and HTTP or HTTPS prefix :param str cert: Absolute path to a client certificate file :param str key: Absolute path to client certificate key file :param str pem: Absolute path to client certificate, as a .pem file :param str ca: Absolute path to CA certs directory to treat as valid or CA bundle file :param bool aad: Use Azure Active Directory for authentication :param bool no_verify: Disable verification for certificates when using HTTPS, note: this is an insecure option and should not be used for production environments """ # Regarding c_rehash: # The c_rehash is needed when specifying a CA certs directory # because requests.Sessions which is used underneath requires # the c_rehash operation to be performed. # See http://docs.python-requests.org/en/master/user/advanced/ from sfctl.config import (set_ca_cert, set_auth, set_aad_cache, set_cluster_endpoint, set_no_verify) from msrest import ServiceClient, Configuration from sfctl.auth import ClientCertAuthentication, AdalAuthentication select_arg_verify(endpoint, cert, key, pem, ca, aad, no_verify) if aad: new_token, new_cache = get_aad_token(endpoint, no_verify) set_aad_cache(new_token, new_cache) rest_client = ServiceClient(AdalAuthentication(no_verify), Configuration(endpoint)) # Make sure basic GET request succeeds rest_client.send(rest_client.get('/')).raise_for_status() else: client_cert = None if pem: client_cert = pem elif cert: client_cert = (cert, key) rest_client = ServiceClient( ClientCertAuthentication(client_cert, ca, no_verify), Configuration(endpoint)) # Make sure basic GET request succeeds rest_client.send(rest_client.get('/')).raise_for_status() set_cluster_endpoint(endpoint) set_no_verify(no_verify) set_ca_cert(ca) set_auth(pem, cert, key, aad)
class TestRuntimeRetry(unittest.TestCase): def setUp(self): cfg = Configuration("https://my_service.com") cfg.retry_policy.backoff_factor=0 creds = Authentication() self.client = ServiceClient(creds, cfg) url = self.client.format_url("/get_endpoint") self.request = self.client.get(url, {'check':True}) return super(TestRuntimeRetry, self).setUp() @httpretty.activate def test_request_retry_502(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=502), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 202, msg="Should retry on 502") @httpretty.activate def test_request_retry_408(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=408), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 202, msg="Should retry on 408") @httpretty.activate def test_request_retry_3_times(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=502), httpretty.Response(body="retry response", status=502), httpretty.Response(body="retry response", status=502), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 202, msg="Should retry 3 times") @httpretty.activate def test_request_retry_max(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=502), httpretty.Response(body="retry response", status=502), httpretty.Response(body="retry response", status=502), httpretty.Response(body="retry response", status=502), ]) with self.assertRaises(ClientRequestError, msg="Max retries reached"): response = self.client.send(self.request) @httpretty.activate def test_request_retry_404(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=404), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 404, msg="Shouldn't retry on 404") @httpretty.activate def test_request_retry_501(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=501), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 501, msg="Shouldn't retry on 501") @httpretty.activate def test_request_retry_505(self): httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="retry response", status=505), httpretty.Response(body='success response', status=202), ]) response = self.client.send(self.request) self.assertEqual(response.status_code, 505, msg="Shouldn't retry on 505")
def test_client_request(self): client = ServiceClient(self.creds, self.cfg) obj = client.get() self.assertEqual(obj.method, 'GET') self.assertIsNone(obj.url) self.assertEqual(obj.params, {}) obj = client.get("/service", {'param':"testing"}) self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service?param=testing") self.assertEqual(obj.params, {}) obj = client.get("service 2") self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service 2") self.cfg.base_url = "https://my_endpoint.com/" obj = client.get("//service3") self.assertEqual(obj.method, 'GET') self.assertEqual(obj.url, "https://my_endpoint.com/service3") obj = client.put() self.assertEqual(obj.method, 'PUT') obj = client.post() self.assertEqual(obj.method, 'POST') obj = client.head() self.assertEqual(obj.method, 'HEAD') obj = client.merge() self.assertEqual(obj.method, 'MERGE') obj = client.patch() self.assertEqual(obj.method, 'PATCH') obj = client.delete() self.assertEqual(obj.method, 'DELETE')
class TestRedirect(unittest.TestCase): def setUp(self): cfg = Configuration("https://my_service.com") cfg.retry_policy.backoff_factor=0 cfg.redirect_policy.max_redirects=2 creds = Authentication() self.client = ServiceClient(creds, cfg) return super(TestRedirect, self).setUp() @httpretty.activate def test_request_redirect_post(self): url = self.client.format_url("/get_endpoint") request = self.client.post(url, {'check':True}) httpretty.register_uri(httpretty.GET, 'https://my_service.com/http/success/get/200', status=200) httpretty.register_uri(httpretty.POST, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=303, method='POST', location='/http/success/get/200'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 200, msg="Should redirect with GET on 303 with location header") self.assertEqual(response.request.method, 'GET') self.assertEqual(response.history[0].status_code, 303) self.assertTrue(response.history[0].is_redirect) httpretty.reset() httpretty.register_uri(httpretty.POST, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=303, method='POST'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 303, msg="Should not redirect on 303 without location header") self.assertEqual(response.history, []) self.assertFalse(response.is_redirect) @httpretty.activate def test_request_redirect_head(self): url = self.client.format_url("/get_endpoint") request = self.client.head(url, {'check':True}) httpretty.register_uri(httpretty.HEAD, 'https://my_service.com/http/success/200', status=200) httpretty.register_uri(httpretty.HEAD, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=307, method='HEAD', location='/http/success/200'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 200, msg="Should redirect on 307 with location header") self.assertEqual(response.request.method, 'HEAD') self.assertEqual(response.history[0].status_code, 307) self.assertTrue(response.history[0].is_redirect) httpretty.reset() httpretty.register_uri(httpretty.HEAD, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=307, method='HEAD'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 307, msg="Should not redirect on 307 without location header") self.assertEqual(response.history, []) self.assertFalse(response.is_redirect) @httpretty.activate def test_request_redirect_delete(self): url = self.client.format_url("/get_endpoint") request = self.client.delete(url, {'check':True}) httpretty.register_uri(httpretty.DELETE, 'https://my_service.com/http/success/200', status=200) httpretty.register_uri(httpretty.DELETE, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=307, method='DELETE', location='/http/success/200'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 200, msg="Should redirect on 307 with location header") self.assertEqual(response.request.method, 'DELETE') self.assertEqual(response.history[0].status_code, 307) self.assertTrue(response.history[0].is_redirect) httpretty.reset() httpretty.register_uri(httpretty.DELETE, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=307, method='DELETE'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 307, msg="Should not redirect on 307 without location header") self.assertEqual(response.history, []) self.assertFalse(response.is_redirect) @httpretty.activate def test_request_redirect_put(self): url = self.client.format_url("/get_endpoint") request = self.client.put(url, {'check':True}) httpretty.register_uri(httpretty.PUT, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=305, method='PUT', location='/http/success/200'), ]) response = self.client.send(request) self.assertEqual(response.status_code, 305, msg="Should not redirect on 305") self.assertEqual(response.history, []) self.assertFalse(response.is_redirect) @httpretty.activate def test_request_redirect_get(self): url = self.client.format_url("/get_endpoint") request = self.client.get(url, {'check':True}) httpretty.register_uri(httpretty.GET, "https://my_service.com/http/finished", responses=[ httpretty.Response(body="", status=200, method='GET'), ]) httpretty.register_uri(httpretty.GET, "https://my_service.com/http/redirect3", responses=[ httpretty.Response(body="", status=307, method='GET', location='/http/finished'), ]) httpretty.register_uri(httpretty.GET, "https://my_service.com/http/redirect2", responses=[ httpretty.Response(body="", status=307, method='GET', location='/http/redirect3'), ]) httpretty.register_uri(httpretty.GET, "https://my_service.com/http/redirect1", responses=[ httpretty.Response(body="", status=307, method='GET', location='/http/redirect2'), ]) httpretty.register_uri(httpretty.GET, "https://my_service.com/get_endpoint", responses=[ httpretty.Response(body="", status=307, method='GET', location='/http/redirect1'), ]) with self.assertRaises(ClientRequestError, msg="Should exceed maximum redirects"): response = self.client.send(request)
def test_request_proxy(self): cfg = Configuration("http://my_service.com") cfg.proxies.add("http://my_service.com", 'http://localhost:57979') creds = Authentication() def hook(adptr, request, *args, **kwargs): self.assertEqual(kwargs.get('proxies'), {"http://my_service.com":'http://localhost:57979'}) kwargs['result']._content_consumed = True kwargs['result'].status_code = 200 return kwargs['result'] client = ServiceClient(creds, cfg) client.add_hook('request', hook, precall=False, overwrite=True) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) os.environ['HTTPS_PROXY'] = "http://localhost:1987" def hook2(adptr, request, *args, **kwargs): self.assertEqual(kwargs.get('proxies')['https'], "http://localhost:1987") kwargs['result']._content_consumed = True kwargs['result'].status_code = 200 return kwargs['result'] cfg = Configuration("http://my_service.com") client = ServiceClient(creds, cfg) client.add_hook('request', hook2, precall=False, overwrite=True) url = client.format_url("/get_endpoint") request = client.get(url, {'check':True}) response = client.send(request) del os.environ['HTTPS_PROXY']