def test_secure_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument
# given
blob_path = "https://kfsecured.blob.core.windows.net/tensorrt/simple_string/"
mock_blob = mock_storage.return_value
mock_blob.list_blobs.side_effect = AzureMissingResourceHttpError(
"fail auth", 404)
mock_get_token.return_value = "some_token"
# when
with pytest.raises(AzureMissingResourceHttpError):
seldon_core.Storage._download_blob(blob_path, "dest_path")
# then
mock_get_token.assert_called()
arg_list = []
for call in mock_storage.call_args_list:
_, kwargs = call
arg_list.append(kwargs)
assert arg_list == [
{
"account_name": "kfsecured"
},
{
"account_name": "kfsecured",
"token_credential": "some_token"
},
]
def test_write_file_consistency_present(self, create_mock,
create_share_mock,
put_file_from_path_mock,
get_file_properties_mock):
from azure.common import AzureMissingResourceHttpError
# given
items_to_write = self.get_batch()
options = self.get_writer_config()
options['options']['check_consistency'] = True
get_file_properties_mock.side_effect = AzureMissingResourceHttpError(
'', 404)
# when:
writer = AzureFileWriter(options, meta())
try:
writer.write_batch(items_to_write)
writer.flush()
finally:
writer.close()
with self.assertRaisesRegexp(InconsistentWriteState, 'Missing file'):
writer.finish_writing()
# then:
self.assertEqual(writer.get_metadata('items_count'), 2)
def _insert(self, operation):
try:
for variant in self._operation_varients:
to_insert = operation.copy()
to_insert.update(self._operation_prep[variant](to_insert))
if not to_insert["PartitionKey"]:
raise AzureMissingResourceHttpError()
if not to_insert["RowKey"]:
raise AzureMissingResourceHttpError()
logging.getLogger(__name__).debug("_insert: Table " + self._operation_tables[variant] + " PartitionKey " + to_insert["PartitionKey"] + " " + to_insert["RowKey"])
self._service.insert_entity(
self._operation_tables[variant],
to_insert
)
except AzureConflictHttpError:
raise DuplicateOperationException()
def test_delete_doesnt_exist(default_storage):
delete_blob = Mock(
side_effect=AzureMissingResourceHttpError('Missing', 404))
default_storage._blob_service.delete_blob = delete_blob
default_storage.delete('something')
delete_blob.assert_called_with('test-container', 'something')
def test_root_invalid_revision(self, mock_get_entity):
"""Test the application returns a 404 response when a specified revision can't be found"""
mock_get_entity.side_effect = AzureMissingResourceHttpError(
"Not Found", 404)
url = '/?{0}=InvalidRevsion'.format(TestConfig.REVISION_PARAMETER)
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
def test_get_first_available_lun(self):
self.service.get_data_disk.side_effect = iter([
self.__create_mock_data_disk(0),
self.__create_mock_data_disk(1),
AzureMissingResourceHttpError('NOT FOUND', 404)
])
result = self.data_disk._DataDisk__get_first_available_lun(
self.cloud_service_name, self.instance_name)
assert self.service.get_data_disk.call_count == 3
assert result == 2 # 0 and 1 are taken
def file_missing_exception(raise_except=False):
storage_provider = os.environ['STORAGE_PROVIDER']
if storage_provider == "AZURE":
return AzureMissingResourceHttpError(
"Missing resource!",
404) if raise_except else AzureMissingResourceHttpError
elif storage_provider == "S3":
return Exception
elif storage_provider == "GCP":
return Exception
def test_attach_without_lun(self, mock_datetime):
# mock no data disks attached has to result in lun 0 assigned later
self.service.get_data_disk.side_effect = AzureMissingResourceHttpError(
'NOT FOUND', 404)
mock_datetime.isoformat.return_value = '0'
self.service.add_data_disk.return_value = self.my_request
result = self.data_disk.attach(self.disk_name, self.cloud_service_name)
self.service.add_data_disk.assert_called_once_with(
self.cloud_service_name,
self.cloud_service_name,
self.cloud_service_name,
0,
disk_name=self.disk_name)
def test_delete_blobs(self):
blob_names = ['to/blob1.txt', 'to/blob2.txt', 'to/blob3.txt']
self.azure_handler.delete_blobs('test', blob_names)
self.mock_blob_service.delete_blob.assert_has_calls([
mock.call('test', 'to/blob1.txt'),
mock.call('test', 'to/blob2.txt'),
mock.call('test', 'to/blob3.txt'),
])
self.mock_azure.reset_mock()
self.mock_blob_service.delete_blob = mock.Mock(
side_effect=AzureMissingResourceHttpError('A', 'B')) # 判断抛错
six.assertCountEqual(
self, self.azure_handler.delete_blobs('test', blob_names), [])
def test_cloud_service(self):
mock = Mock()
self.service.service.get_hosted_service_properties = mock
mock.return_value = True
self.assertTrue(
self.service.cloud_service_exists(test_conf.meanless_name))
mock.return_value = None
self.assertFalse(
self.service.cloud_service_exists(test_conf.meanless_name))
mock.side_effect = AzureMissingResourceHttpError(233, 233)
self.assertFalse(
self.service.cloud_service_exists(test_conf.meanless_name))
def test_list_blobs(self):
# Since “name” is an argument to the Mock constructor,
# if you want your mock object to have a “name” attribute
# you can’t just pass it in at creation time.
mock_obj1 = mock.MagicMock()
mock_obj1.name = "a.txt"
mock_obj2 = mock.MagicMock()
mock_obj2.name = "a.wav"
mock_obj3 = mock.MagicMock()
mock_obj3.name = "a.metadata"
self.mock_blob_service.list_blobs.return_value = [
mock_obj1, mock_obj2, mock_obj3
]
six.assertCountEqual(self, self.azure_handler.list_blobs('test'),
["a.txt", "a.wav", "a.metadata"])
self.mock_blob_service.list_blobs.assert_called_with('test',
prefix=None,
timeout=300)
six.assertCountEqual(
self,
self.azure_handler.list_blobs('test',
prefix="test-",
suffix=".wav"), ["a.wav"])
self.mock_blob_service.list_blobs.assert_called_with('test',
prefix="test-",
timeout=300)
six.assertCountEqual(
self, self.azure_handler.list_blobs('test',
suffix=(".wav", ".txt")),
["a.wav", "a.txt"])
self.mock_blob_service.list_blobs.assert_called_with('test',
prefix=None,
timeout=300)
self.mock_azure.reset_mock()
self.mock_blob_service.list_blobs = mock.Mock(
side_effect=AzureMissingResourceHttpError('a', 'b'))
six.assertCountEqual(
self, self.azure_handler.list_blobs('test',
suffix=(".wav", ".txt")), [])
def get_data_batch_from_blob(result_loc_, prefix=None, backup=False):
try:
account_name = os.environ['AZURE_STORAGE_ACCOUNT']
account_key = os.environ['AZURE_STORAGE_ACCESS_KEY']
block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
container_name = 'portal-reports-backup' if backup else 'reports'
blob_list = block_blob_service.list_blobs(container_name,
prefix=prefix)
for blob in blob_list:
blob_result_loc_ = result_loc_.joinpath(blob.name).parent
os.makedirs(blob_result_loc_, exist_ok=True)
block_blob_service.get_blob_to_path(container_name=container_name,
blob_name=blob.name,
file_path=str(result_loc_) +
"/" + blob.name)
except AzureMissingResourceHttpError:
raise AzureMissingResourceHttpError("Missing resource!", 404)
def get_data_from_store(container_name,
blob_name,
file_path,
is_private=False):
try:
if is_private:
account_name = os.environ['AZURE_STORAGE_ACCOUNT']
account_key = os.environ['AZURE_STORAGE_ACCESS_KEY']
else:
account_name = os.environ['PUBLIC_AZURE_STORAGE_ACCOUNT']
account_key = os.environ['PUBLIC_AZURE_STORAGE_ACCESS_KEY']
block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
block_blob_service.get_blob_to_path(container_name=container_name,
blob_name=blob_name,
file_path=file_path)
except AzureMissingResourceHttpError:
raise AzureMissingResourceHttpError("Missing resource!", 404)
except Exception as e:
raise Exception('Could not read from blob!' + str(e))
def get_batch_data_from_blob(result_loc_, prefix_, backup=False):
try:
account_name = os.environ['PUBLIC_AZURE_STORAGE_ACCOUNT']
account_key = os.environ['PUBLIC_AZURE_STORAGE_ACCESS_KEY']
block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
if backup:
container_name = os.environ['REPORT_BACKUP_CONTAINER']
blob_list = block_blob_service.list_blobs(container_name,
prefix=prefix_)
for blob in blob_list:
report_name, date_name, file_name = blob.name.split('/')
if file_name.startswith('content_consumption_lastweek'):
slug = file_name.split('_')[-1].split('.')[0]
result_loc_.joinpath(slug).mkdir(exist_ok=True)
file_path_ = result_loc_.joinpath(
slug, 'content_consumption_lastweek_{}.csv'.format(
date_name))
block_blob_service.get_blob_to_path(
container_name=container_name,
blob_name=blob.name,
file_path=str(file_path_))
except AzureMissingResourceHttpError:
raise AzureMissingResourceHttpError("Missing resource!", 404)
def get_entity(self, table_name, partition_key, key):
full_key = f"{table_name}/{partition_key}/{key}"
try:
return self.data[full_key]
except KeyError:
raise AzureMissingResourceHttpError("Not found", 404)
def given_download_exception(self):
error = AzureMissingResourceHttpError('injected error', 404)
self.email_server_client_mock.download.return_value = ('id', 'folder')
self.azure_client_mock.get_blob_to_stream.side_effect = error