def test_get_queue_client_expired_token(self): """ Exception handler should deal with a bad token by clearing cache and retrying. So if we provide a bad token followed by a real one in our mock, we expect it to end up getting the real token. """ real_token = StorageUtilities.get_storage_token(self.session) with patch('c7n_azure.storage_utils.QueueService.create_queue' ) as create_mock: with patch( 'c7n_azure.storage_utils.StorageUtilities.get_storage_token' ) as token_mock: error = AzureHttpError('', 403) error.error_code = 'AuthenticationFailed' # Two side effects: one with a bad token and an error, # and one with a good token and no error create_mock.side_effect = [error, None] token_mock.side_effect = [TokenCredential('fake'), real_token] url = "https://fake.queue.core.windows.net/testcc" queue_service, queue_name = \ StorageUtilities.get_queue_client_by_uri(url, self.session) # We end up with the real token (after a retry) self.assertEqual(real_token, queue_service.authentication)
def test_create_container_fail(self, mock_make): error = AzureHttpError('because Im a test', 418) error.message = 'This should work' mock_make.side_effect = error url = "/api/v1/project/{0}/azureblobstorage/newcontainer/".format( self.project._id) ret = self.app.post_json(url, {'container_name': 'doesntevenmatter'}, auth=self.user.auth, expect_errors=True) assert_equals( ret.body, '{"message": "This should work", "title": "Problem connecting to Azure Blob Storage"}' )
def _create_or_update(self, defn): info = { 'location': defn.location, 'tags': defn.tags, 'sku': { 'name': "{0}_{1}".format(defn.tier, defn.family), 'tier': defn.tier, 'family': defn.family, }, 'properties': { 'serviceProviderProperties': { 'serviceProviderName': defn.service_provider_name, 'peeringLocation': defn.peering_location, 'bandwidthInMbps': defn.bandwidth, }, 'peerings': [ { 'name': _n, 'properties': _p, } for _n, _p in defn.peerings.iteritems() ], }, } http_request = self.mk_request('PUT') http_request.data = json.dumps(info) http_request.headers['Content-Length'] = len(http_request.data) response = self.nrpc().send_request(http_request) if response.status_code not in [200, 201, 202]: raise AzureHttpError(response.content, response.status_code) self.get_settled_resource() self.state = self.UP self.copy_properties(defn)
def get_token(): """ generate or retrieve token for azure integration base on https://developer.microsoft.com/en-us/graph/docs/concepts/auth_overview """ logger.info('Request for token started') token = cache.get(AZURE_GRAPH_API_TOKEN_CACHE_KEY) if not token: logger.info('Request new token to be generated') path = settings.AZURE_TOKEN_URL post_dict = { 'grant_type': 'client_credentials', 'client_id': settings.AZURE_CLIENT_ID, 'client_secret': settings.AZURE_CLIENT_SECRET, 'resource': settings.AZURE_GRAPH_API_BASE_URL } response = requests.post(path, post_dict) if response.status_code == 200: jresponse = response.json() token = jresponse['access_token'] # Cache token for 3600 seconds, which matches the default Azure token expiration cache.set(AZURE_GRAPH_API_TOKEN_CACHE_KEY, token, 3600) logger.info('Token retrieved') else: logger.error('Error during token retrieval') raise AzureHttpError( 'Error during token retrieval {}'.format(response.status_code), response.status_code) return token
def _http_error_handler(http_error): ''' Simple error handler for azure.''' message = str(http_error) error_code = None if 'x-ms-error-code' in http_error.respheader: error_code = http_error.respheader['x-ms-error-code'] message += ' ErrorCode: ' + error_code if http_error.respbody is not None: message += '\n' + http_error.respbody.decode('utf-8-sig') ex = AzureHttpError(message, http_error.status) ex.error_code = error_code raise ex
def test_container_does_not_exists(self): self.azure_storage.block_blob_service.configure_mock( **{ 'exists.side_effect': AzureHttpError("not found", 404), } ) exists = self.azure_storage.container_exists(BERGAMOT_CONTAINER_NAME) self.assertEqual(exists, False)
def test_blob_does_not_exists(self): self.azure_storage.block_blob_service.configure_mock( **{ 'get_container_properties.side_effect': AzureHttpError("not found", 404), } ) file_size = 100 exists = self.azure_storage.blob_exists(BERGAMOT_CONTAINER_NAME, BLOB_NAME, file_size) self.assertEqual(exists, False)
def _http_error_handler(http_error): ''' Simple error handler for azure.''' message = str(http_error) respbody = http_error.respbody if respbody is not None: if hasattr(http_error, "decode"): message += '\n' + respbody.decode('utf-8-sig') else: message += '\n' + respbody raise AzureHttpError(message, http_error.status)
def test_handle_azure_exception(): """Test airfs.storage.azure._handle_azure_exception""" from airfs.storage.azure import _handle_azure_exception from azure.common import AzureHttpError # type: ignore from airfs._core.exceptions import ObjectNotFoundError, ObjectPermissionError # Any error with pytest.raises(AzureHttpError): with _handle_azure_exception(): raise AzureHttpError(message="", status_code=400) # 404 error with pytest.raises(ObjectNotFoundError): with _handle_azure_exception(): raise AzureHttpError(message="", status_code=404) # 403 error with pytest.raises(ObjectPermissionError): with _handle_azure_exception(): raise AzureHttpError(message="", status_code=403)
def test_access_error(self, logger_mock, get_queue_client_by_uri, put_queue_message): put_queue_message.side_effect = AzureHttpError('forbidden', 403) get_queue_client_by_uri.return_value = 'service', 'name' action = Notify() action.send_to_azure_queue("url", "message", local_session(Session)) args, _ = logger_mock.call_args self.assertIsNotNone(re.match("Access Error*", args[0]))
def test_update_policies_list_blobs_azure_http_error( self, get_blob_client_mock, _2, _3, _4, _5): client_mock = Mock() client_mock.list_blobs.side_effect = AzureHttpError( "failed to list blobs", 400) get_blob_client_mock.return_value = (client_mock, None, None) host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME, DEFAULT_POLICY_STORAGE) with self.assertRaises(AzureHttpError): host.update_policies() client_mock.list_blobs.assert_called()
def test_handle_azure_exception(): """Test pycosio.storage.azure._handle_azure_exception""" from pycosio.storage.azure import _handle_azure_exception from azure.common import AzureHttpError from pycosio._core.exceptions import (ObjectNotFoundError, ObjectPermissionError) # Any error with pytest.raises(AzureHttpError): with _handle_azure_exception(): raise AzureHttpError(message='', status_code=400) # 404 error with pytest.raises(ObjectNotFoundError): with _handle_azure_exception(): raise AzureHttpError(message='', status_code=404) # 403 error with pytest.raises(ObjectPermissionError): with _handle_azure_exception(): raise AzureHttpError(message='', status_code=403)
def test_write_raises(self): handler = WasbTaskHandler( self.local_log_location, self.wasb_log_folder, self.container_name, self.filename_template, True ) with mock.patch.object(handler.log, 'error') as mock_error: with mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook") as mock_hook: mock_hook.return_value.load_string.side_effect = AzureHttpError("failed to connect", 404) handler.wasb_write('text', self.remote_log_location, append=False) mock_error.assert_called_once_with( 'Could not write logs to %s', 'remote/log/location/1.log', exc_info=True )
def _create_or_update(self, defn): info = {"location": "global", "tags": defn.tags, "properties": {}} http_request = self.mk_request('PUT') http_request.data = json.dumps(info) http_request.headers['Content-Length'] = len(http_request.data) response = self.nrpc().send_request(http_request) if response.status_code not in [200, 201]: raise AzureHttpError(response.content, response.status_code) self.state = self.UP self.copy_properties(defn)
def test_workflow_exception_when_storage_coordinates_raises_http_error( self): self.mock_tf_call_service.run.return_value = \ {(1536, 8192): [Rect(562, 954, 665, 111, 0.3364628255367279)]} self.mock_coordinate_storage.store.side_effect = AzureHttpError('', '') self.workflow = WorkFlow( self.graph, self.chunk_image, self.mock_tf_call_service, self.apply_coordinate_correction, self.filter_merge_boxes, self.mock_label_service, self.mock_coordinate_storage, self.mock_post_processing, self.provider) self.workflow.run() self.workflow.provider.acknowledge.assert_not_called()
def __get_blob_to_path(self, container_name, blob_name, destination_file_path, username=None, password=None, ipaddress=None, vhost=None): """ download data from blob storage :param container_name: blob container name :param blob_name: blob path :param destination_file_path: path to download the file to :param username: username for rabbitmq instance :param password: password for rabbitmq instance :param ipaddress: ip for rabbitmq instance :param vhost: rabbitmq vhost :return: azure.storage.blob.baseblobservice.Blob instance with content properties and metadata """ blob_name = self.__format_blob_name(blob_name) if username: queue = "{}_pull".format(self.account_name) semaphore = RabbitMqSemaphore( username, password, ipaddress, queue, vhost, self.blob_client.get_blob_to_path, [container_name, blob_name, destination_file_path], {}) blob = semaphore.run() else: try: blob_client = self.blob_client.get_blob_client( container_name, blob_name) with open(destination_file_path, "wb") as my_blob: download_stream = blob_client.download_blob() my_blob.write(download_stream.readall()) blob = blob_client.get_blob_properties() except Exception as exc: self.logger.exception("Error downloading {} from {}".format( blob_name, container_name)) raise exc if not blob: self.logger.exception('Blob SDK returned None, retrying') raise AzureHttpError('Blob download failure', 1) return blob
def test_hook_raises(self): handler = WasbTaskHandler( self.local_log_location, self.wasb_log_folder, self.container_name, self.filename_template, True ) with mock.patch.object(handler.log, 'error') as mock_error: with mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook") as mock_hook: mock_hook.side_effect = AzureHttpError("failed to connect", 404) # Initialize the hook handler.hook mock_error.assert_called_once_with( 'Could not create an WasbHook with connection id "%s". ' 'Please make sure that airflow[azure] is installed and ' 'the Wasb connection exists.', "wasb_default", )
def test_wasb_read_raises(self): handler = WasbTaskHandler(self.local_log_location, self.wasb_log_folder, self.container_name, self.filename_template, True) with mock.patch.object(handler.log, 'error') as mock_error: with mock.patch( "airflow.providers.microsoft.azure.hooks.wasb.WasbHook" ) as mock_hook: mock_hook.return_value.read_file.side_effect = AzureHttpError( "failed to connect", 404) handler.wasb_read(self.remote_log_location, return_error=True) mock_error.assert_called_once_with( "Message: '%s', exception '%s'", 'Could not read logs from remote/log/location/1.log', ANY, exc_info=True, )
def test_creat_hosted_service(self): mock_create = Mock() mock_wait = Mock() self.service.service.create_cloud_service = mock_create self.service.service.wait_for_operation_status = mock_wait mock_create.return_value = Context(request_id=test_conf.meanless_id) mock_wait.return_value = Context(status=ASYNC_OP_RESULT.SUCCEEDED) self.assertTrue( self.service.create_cloud_service(test_conf.meanless_name, test_conf.meanless_name, test_conf.meanless_name)) mock_create.side_effect = AzureHttpError(233, 233) self.assertFalse( self.service.create_cloud_service(test_conf.meanless_name, test_conf.meanless_name, test_conf.meanless_name))
def analyse_page(url, access_token): """ analyse the page :param url: url to call :param access_token: azure access token :return: tuple, next page url and delta url """ headers = {'Authorization': 'Bearer {}'.format(access_token)} response = requests.get(url, headers=headers) jresponse = response.json() if response.status_code == 200: logger.info('Azure: Information retrieved') handle_records(jresponse) url = jresponse.get('@odata.nextLink', None) else: logger.error('Error during synchronization process') raise AzureHttpError( 'Error processing the response {}'.format(response.status_code), response.status_code) return url, jresponse.get('@odata.deltaLink', None)
def test_error_writing_to_blob(self, logger_mock): AzureStorageOutput.get_blob_client_wrapper = gm = mock.MagicMock() gm.return_value = None, "logs", 'xyz' output = self.get_azure_output() output.blob_service = mock.MagicMock() output.blob_service.create_blob_from_path = mock.MagicMock() output.blob_service.create_blob_from_path.side_effect = AzureHttpError('not found', 404) # Generate fake output file with open(os.path.join(output.root_dir, "foo.txt"), "w") as fh: fh.write("abc") output.upload() args, _ = logger_mock.call_args self.assertIsNone(re.match("Access Error*", args[0])) self.assertIsNotNone(re.match("Error*", args[0]))
def _create_or_update(self, defn): info = { 'location': "global", 'tags': defn.tags, 'properties': { 'profileStatus': defn.enable, 'trafficRoutingMethod': defn.traffic_routing_method, 'dnsConfig': { 'relativeName': defn.relative_name, 'ttl': defn.ttl, }, 'monitorConfig': { 'protocol': defn.protocol, 'port': defn.port, 'path': defn.path, }, 'endpoints': [{ 'name': _n, 'type': "Microsoft.Network/TrafficManagerProfiles/ExternalEndpoints", 'properties': _ep } for _n, _ep in defn.endpoints.iteritems()], }, } http_request = self.mk_request('PUT') http_request.data = json.dumps(info) http_request.headers['Content-Length'] = len(http_request.data) response = self.nrpc().send_request(http_request) if response.status_code not in [200, 201]: raise AzureHttpError(response.content, response.status_code) self.state = self.UP self.copy_properties(defn) r = self.get_settled_resource() or {} fqdn = r.get('properties', {}).get('dnsConfig', {}).get('fqdn', None) self.log('FQDN: {0}'.format(fqdn))
def raise_500(): """Raise 500 error""" raise AzureHttpError(message="", status_code=500)
def create_blob_from_bytes(_1, _2, _3, **kwargs): raise AzureHttpError("Failed to create blob", 403)
def _general_error_handler(http_error): ''' Simple error handler for azure.''' message = str(http_error) if http_error.respbody is not None: message += '\n' + http_error.respbody.decode('utf-8-sig') raise AzureHttpError(message, http_error.status)
def destroy_resource(self): response = self.nrpc().send_request(self.mk_request('DELETE')) if response.status_code != 200: raise AzureHttpError(response.content, response.status_code)
def destroy_resource(self): response = self.nrpc().send_request(self.mk_request('DELETE')) if response.status_code not in [200, 202, 204]: raise AzureHttpError(response.content, response.status_code) self.get_settled_resource() # wait for the delete operation to finish
def raise_404(): """Raise 404 error""" raise AzureHttpError(message="", status_code=404)
def raise_416(): """Raise 416 error""" raise AzureHttpError(message="", status_code=416)