def test_get_queue_client_expired_token(self):
"""
Exception handler should deal with a bad token by clearing
cache and retrying. So if we provide a bad token followed
by a real one in our mock, we expect it to end up getting
the real token.
"""
real_token = StorageUtilities.get_storage_token(self.session)
with patch('c7n_azure.storage_utils.QueueService.create_queue'
) as create_mock:
with patch(
'c7n_azure.storage_utils.StorageUtilities.get_storage_token'
) as token_mock:
error = AzureHttpError('', 403)
error.error_code = 'AuthenticationFailed'
# Two side effects: one with a bad token and an error,
# and one with a good token and no error
create_mock.side_effect = [error, None]
token_mock.side_effect = [TokenCredential('fake'), real_token]
url = "https://fake.queue.core.windows.net/testcc"
queue_service, queue_name = \
StorageUtilities.get_queue_client_by_uri(url, self.session)
# We end up with the real token (after a retry)
self.assertEqual(real_token, queue_service.authentication)
def test_create_container_fail(self, mock_make):
error = AzureHttpError('because Im a test', 418)
error.message = 'This should work'
mock_make.side_effect = error
url = "/api/v1/project/{0}/azureblobstorage/newcontainer/".format(
self.project._id)
ret = self.app.post_json(url, {'container_name': 'doesntevenmatter'},
auth=self.user.auth,
expect_errors=True)
assert_equals(
ret.body,
'{"message": "This should work", "title": "Problem connecting to Azure Blob Storage"}'
)
def _create_or_update(self, defn):
info = {
'location': defn.location,
'tags': defn.tags,
'sku': {
'name': "{0}_{1}".format(defn.tier, defn.family),
'tier': defn.tier,
'family': defn.family,
},
'properties': {
'serviceProviderProperties': {
'serviceProviderName': defn.service_provider_name,
'peeringLocation': defn.peering_location,
'bandwidthInMbps': defn.bandwidth,
},
'peerings': [
{ 'name': _n,
'properties': _p,
} for _n, _p in defn.peerings.iteritems()
],
},
}
http_request = self.mk_request('PUT')
http_request.data = json.dumps(info)
http_request.headers['Content-Length'] = len(http_request.data)
response = self.nrpc().send_request(http_request)
if response.status_code not in [200, 201, 202]:
raise AzureHttpError(response.content, response.status_code)
self.get_settled_resource()
self.state = self.UP
self.copy_properties(defn)
def get_token():
"""
generate or retrieve token for azure integration
base on https://developer.microsoft.com/en-us/graph/docs/concepts/auth_overview
"""
logger.info('Request for token started')
token = cache.get(AZURE_GRAPH_API_TOKEN_CACHE_KEY)
if not token:
logger.info('Request new token to be generated')
path = settings.AZURE_TOKEN_URL
post_dict = {
'grant_type': 'client_credentials',
'client_id': settings.AZURE_CLIENT_ID,
'client_secret': settings.AZURE_CLIENT_SECRET,
'resource': settings.AZURE_GRAPH_API_BASE_URL
}
response = requests.post(path, post_dict)
if response.status_code == 200:
jresponse = response.json()
token = jresponse['access_token']
# Cache token for 3600 seconds, which matches the default Azure token expiration
cache.set(AZURE_GRAPH_API_TOKEN_CACHE_KEY, token, 3600)
logger.info('Token retrieved')
else:
logger.error('Error during token retrieval')
raise AzureHttpError(
'Error during token retrieval {}'.format(response.status_code),
response.status_code)
return token
def _http_error_handler(http_error):
''' Simple error handler for azure.'''
message = str(http_error)
error_code = None
if 'x-ms-error-code' in http_error.respheader:
error_code = http_error.respheader['x-ms-error-code']
message += ' ErrorCode: ' + error_code
if http_error.respbody is not None:
message += '\n' + http_error.respbody.decode('utf-8-sig')
ex = AzureHttpError(message, http_error.status)
ex.error_code = error_code
raise ex
def test_container_does_not_exists(self):
self.azure_storage.block_blob_service.configure_mock(
**{
'exists.side_effect': AzureHttpError("not found", 404),
}
)
exists = self.azure_storage.container_exists(BERGAMOT_CONTAINER_NAME)
self.assertEqual(exists, False)
def test_blob_does_not_exists(self):
self.azure_storage.block_blob_service.configure_mock(
**{
'get_container_properties.side_effect': AzureHttpError("not found", 404),
}
)
file_size = 100
exists = self.azure_storage.blob_exists(BERGAMOT_CONTAINER_NAME, BLOB_NAME, file_size)
self.assertEqual(exists, False)
def _http_error_handler(http_error):
''' Simple error handler for azure.'''
message = str(http_error)
respbody = http_error.respbody
if respbody is not None:
if hasattr(http_error, "decode"):
message += '\n' + respbody.decode('utf-8-sig')
else:
message += '\n' + respbody
raise AzureHttpError(message, http_error.status)
def test_handle_azure_exception():
"""Test airfs.storage.azure._handle_azure_exception"""
from airfs.storage.azure import _handle_azure_exception
from azure.common import AzureHttpError # type: ignore
from airfs._core.exceptions import ObjectNotFoundError, ObjectPermissionError
# Any error
with pytest.raises(AzureHttpError):
with _handle_azure_exception():
raise AzureHttpError(message="", status_code=400)
# 404 error
with pytest.raises(ObjectNotFoundError):
with _handle_azure_exception():
raise AzureHttpError(message="", status_code=404)
# 403 error
with pytest.raises(ObjectPermissionError):
with _handle_azure_exception():
raise AzureHttpError(message="", status_code=403)
def test_access_error(self, logger_mock, get_queue_client_by_uri,
put_queue_message):
put_queue_message.side_effect = AzureHttpError('forbidden', 403)
get_queue_client_by_uri.return_value = 'service', 'name'
action = Notify()
action.send_to_azure_queue("url", "message", local_session(Session))
args, _ = logger_mock.call_args
self.assertIsNotNone(re.match("Access Error*", args[0]))
def test_update_policies_list_blobs_azure_http_error(
self, get_blob_client_mock, _2, _3, _4, _5):
client_mock = Mock()
client_mock.list_blobs.side_effect = AzureHttpError(
"failed to list blobs", 400)
get_blob_client_mock.return_value = (client_mock, None, None)
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
with self.assertRaises(AzureHttpError):
host.update_policies()
client_mock.list_blobs.assert_called()
def test_handle_azure_exception():
"""Test pycosio.storage.azure._handle_azure_exception"""
from pycosio.storage.azure import _handle_azure_exception
from azure.common import AzureHttpError
from pycosio._core.exceptions import (ObjectNotFoundError,
ObjectPermissionError)
# Any error
with pytest.raises(AzureHttpError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=400)
# 404 error
with pytest.raises(ObjectNotFoundError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=404)
# 403 error
with pytest.raises(ObjectPermissionError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=403)
def test_write_raises(self):
handler = WasbTaskHandler(
self.local_log_location, self.wasb_log_folder, self.container_name, self.filename_template, True
)
with mock.patch.object(handler.log, 'error') as mock_error:
with mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook") as mock_hook:
mock_hook.return_value.load_string.side_effect = AzureHttpError("failed to connect", 404)
handler.wasb_write('text', self.remote_log_location, append=False)
mock_error.assert_called_once_with(
'Could not write logs to %s', 'remote/log/location/1.log', exc_info=True
)
def _create_or_update(self, defn):
info = {"location": "global", "tags": defn.tags, "properties": {}}
http_request = self.mk_request('PUT')
http_request.data = json.dumps(info)
http_request.headers['Content-Length'] = len(http_request.data)
response = self.nrpc().send_request(http_request)
if response.status_code not in [200, 201]:
raise AzureHttpError(response.content, response.status_code)
self.state = self.UP
self.copy_properties(defn)
def test_workflow_exception_when_storage_coordinates_raises_http_error(
self):
self.mock_tf_call_service.run.return_value = \
{(1536, 8192):
[Rect(562, 954, 665, 111, 0.3364628255367279)]}
self.mock_coordinate_storage.store.side_effect = AzureHttpError('', '')
self.workflow = WorkFlow(
self.graph, self.chunk_image, self.mock_tf_call_service,
self.apply_coordinate_correction, self.filter_merge_boxes,
self.mock_label_service, self.mock_coordinate_storage,
self.mock_post_processing, self.provider)
self.workflow.run()
self.workflow.provider.acknowledge.assert_not_called()
def __get_blob_to_path(self,
container_name,
blob_name,
destination_file_path,
username=None,
password=None,
ipaddress=None,
vhost=None):
"""
download data from blob storage
:param container_name: blob container name
:param blob_name: blob path
:param destination_file_path: path to download the file to
:param username: username for rabbitmq instance
:param password: password for rabbitmq instance
:param ipaddress: ip for rabbitmq instance
:param vhost: rabbitmq vhost
:return: azure.storage.blob.baseblobservice.Blob instance with content properties and metadata
"""
blob_name = self.__format_blob_name(blob_name)
if username:
queue = "{}_pull".format(self.account_name)
semaphore = RabbitMqSemaphore(
username, password, ipaddress, queue, vhost,
self.blob_client.get_blob_to_path,
[container_name, blob_name, destination_file_path], {})
blob = semaphore.run()
else:
try:
blob_client = self.blob_client.get_blob_client(
container_name, blob_name)
with open(destination_file_path, "wb") as my_blob:
download_stream = blob_client.download_blob()
my_blob.write(download_stream.readall())
blob = blob_client.get_blob_properties()
except Exception as exc:
self.logger.exception("Error downloading {} from {}".format(
blob_name, container_name))
raise exc
if not blob:
self.logger.exception('Blob SDK returned None, retrying')
raise AzureHttpError('Blob download failure', 1)
return blob
def test_hook_raises(self):
handler = WasbTaskHandler(
self.local_log_location, self.wasb_log_folder, self.container_name, self.filename_template, True
)
with mock.patch.object(handler.log, 'error') as mock_error:
with mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook") as mock_hook:
mock_hook.side_effect = AzureHttpError("failed to connect", 404)
# Initialize the hook
handler.hook
mock_error.assert_called_once_with(
'Could not create an WasbHook with connection id "%s". '
'Please make sure that airflow[azure] is installed and '
'the Wasb connection exists.',
"wasb_default",
)
def test_wasb_read_raises(self):
handler = WasbTaskHandler(self.local_log_location,
self.wasb_log_folder, self.container_name,
self.filename_template, True)
with mock.patch.object(handler.log, 'error') as mock_error:
with mock.patch(
"airflow.providers.microsoft.azure.hooks.wasb.WasbHook"
) as mock_hook:
mock_hook.return_value.read_file.side_effect = AzureHttpError(
"failed to connect", 404)
handler.wasb_read(self.remote_log_location, return_error=True)
mock_error.assert_called_once_with(
"Message: '%s', exception '%s'",
'Could not read logs from remote/log/location/1.log',
ANY,
exc_info=True,
)
def test_creat_hosted_service(self):
mock_create = Mock()
mock_wait = Mock()
self.service.service.create_cloud_service = mock_create
self.service.service.wait_for_operation_status = mock_wait
mock_create.return_value = Context(request_id=test_conf.meanless_id)
mock_wait.return_value = Context(status=ASYNC_OP_RESULT.SUCCEEDED)
self.assertTrue(
self.service.create_cloud_service(test_conf.meanless_name,
test_conf.meanless_name,
test_conf.meanless_name))
mock_create.side_effect = AzureHttpError(233, 233)
self.assertFalse(
self.service.create_cloud_service(test_conf.meanless_name,
test_conf.meanless_name,
test_conf.meanless_name))
def analyse_page(url, access_token):
"""
analyse the page
:param url: url to call
:param access_token: azure access token
:return: tuple, next page url and delta url
"""
headers = {'Authorization': 'Bearer {}'.format(access_token)}
response = requests.get(url, headers=headers)
jresponse = response.json()
if response.status_code == 200:
logger.info('Azure: Information retrieved')
handle_records(jresponse)
url = jresponse.get('@odata.nextLink', None)
else:
logger.error('Error during synchronization process')
raise AzureHttpError(
'Error processing the response {}'.format(response.status_code),
response.status_code)
return url, jresponse.get('@odata.deltaLink', None)
def test_error_writing_to_blob(self, logger_mock):
AzureStorageOutput.get_blob_client_wrapper = gm = mock.MagicMock()
gm.return_value = None, "logs", 'xyz'
output = self.get_azure_output()
output.blob_service = mock.MagicMock()
output.blob_service.create_blob_from_path = mock.MagicMock()
output.blob_service.create_blob_from_path.side_effect = AzureHttpError('not found', 404)
# Generate fake output file
with open(os.path.join(output.root_dir, "foo.txt"), "w") as fh:
fh.write("abc")
output.upload()
args, _ = logger_mock.call_args
self.assertIsNone(re.match("Access Error*", args[0]))
self.assertIsNotNone(re.match("Error*", args[0]))
def _create_or_update(self, defn):
info = {
'location': "global",
'tags': defn.tags,
'properties': {
'profileStatus':
defn.enable,
'trafficRoutingMethod':
defn.traffic_routing_method,
'dnsConfig': {
'relativeName': defn.relative_name,
'ttl': defn.ttl,
},
'monitorConfig': {
'protocol': defn.protocol,
'port': defn.port,
'path': defn.path,
},
'endpoints': [{
'name': _n,
'type':
"Microsoft.Network/TrafficManagerProfiles/ExternalEndpoints",
'properties': _ep
} for _n, _ep in defn.endpoints.iteritems()],
},
}
http_request = self.mk_request('PUT')
http_request.data = json.dumps(info)
http_request.headers['Content-Length'] = len(http_request.data)
response = self.nrpc().send_request(http_request)
if response.status_code not in [200, 201]:
raise AzureHttpError(response.content, response.status_code)
self.state = self.UP
self.copy_properties(defn)
r = self.get_settled_resource() or {}
fqdn = r.get('properties', {}).get('dnsConfig', {}).get('fqdn', None)
self.log('FQDN: {0}'.format(fqdn))
def raise_500():
"""Raise 500 error"""
raise AzureHttpError(message="", status_code=500)
def create_blob_from_bytes(_1, _2, _3, **kwargs):
raise AzureHttpError("Failed to create blob", 403)
def _general_error_handler(http_error):
''' Simple error handler for azure.'''
message = str(http_error)
if http_error.respbody is not None:
message += '\n' + http_error.respbody.decode('utf-8-sig')
raise AzureHttpError(message, http_error.status)
def destroy_resource(self):
response = self.nrpc().send_request(self.mk_request('DELETE'))
if response.status_code != 200:
raise AzureHttpError(response.content, response.status_code)
def destroy_resource(self):
response = self.nrpc().send_request(self.mk_request('DELETE'))
if response.status_code not in [200, 202, 204]:
raise AzureHttpError(response.content, response.status_code)
self.get_settled_resource() # wait for the delete operation to finish
def raise_404():
"""Raise 404 error"""
raise AzureHttpError(message="", status_code=404)
def raise_416():
"""Raise 416 error"""
raise AzureHttpError(message="", status_code=416)
