def test_secure_blob(mock_storage, mock_get_token, mock_makedirs): # pylint: disable=unused-argument # given blob_path = "https://kfsecured.blob.core.windows.net/tensorrt/simple_string/" mock_blob = mock_storage.return_value mock_blob.list_blobs.side_effect = AzureMissingResourceHttpError( "fail auth", 404) mock_get_token.return_value = "some_token" # when with pytest.raises(AzureMissingResourceHttpError): seldon_core.Storage._download_blob(blob_path, "dest_path") # then mock_get_token.assert_called() arg_list = [] for call in mock_storage.call_args_list: _, kwargs = call arg_list.append(kwargs) assert arg_list == [ { "account_name": "kfsecured" }, { "account_name": "kfsecured", "token_credential": "some_token" }, ]
def test_write_file_consistency_present(self, create_mock, create_share_mock, put_file_from_path_mock, get_file_properties_mock): from azure.common import AzureMissingResourceHttpError # given items_to_write = self.get_batch() options = self.get_writer_config() options['options']['check_consistency'] = True get_file_properties_mock.side_effect = AzureMissingResourceHttpError( '', 404) # when: writer = AzureFileWriter(options, meta()) try: writer.write_batch(items_to_write) writer.flush() finally: writer.close() with self.assertRaisesRegexp(InconsistentWriteState, 'Missing file'): writer.finish_writing() # then: self.assertEqual(writer.get_metadata('items_count'), 2)
def _insert(self, operation): try: for variant in self._operation_varients: to_insert = operation.copy() to_insert.update(self._operation_prep[variant](to_insert)) if not to_insert["PartitionKey"]: raise AzureMissingResourceHttpError() if not to_insert["RowKey"]: raise AzureMissingResourceHttpError() logging.getLogger(__name__).debug("_insert: Table " + self._operation_tables[variant] + " PartitionKey " + to_insert["PartitionKey"] + " " + to_insert["RowKey"]) self._service.insert_entity( self._operation_tables[variant], to_insert ) except AzureConflictHttpError: raise DuplicateOperationException()
def test_delete_doesnt_exist(default_storage): delete_blob = Mock( side_effect=AzureMissingResourceHttpError('Missing', 404)) default_storage._blob_service.delete_blob = delete_blob default_storage.delete('something') delete_blob.assert_called_with('test-container', 'something')
def test_root_invalid_revision(self, mock_get_entity): """Test the application returns a 404 response when a specified revision can't be found""" mock_get_entity.side_effect = AzureMissingResourceHttpError( "Not Found", 404) url = '/?{0}=InvalidRevsion'.format(TestConfig.REVISION_PARAMETER) response = self.client.get(url) self.assertEqual(response.status_code, 404)
def test_get_first_available_lun(self): self.service.get_data_disk.side_effect = iter([ self.__create_mock_data_disk(0), self.__create_mock_data_disk(1), AzureMissingResourceHttpError('NOT FOUND', 404) ]) result = self.data_disk._DataDisk__get_first_available_lun( self.cloud_service_name, self.instance_name) assert self.service.get_data_disk.call_count == 3 assert result == 2 # 0 and 1 are taken
def file_missing_exception(raise_except=False): storage_provider = os.environ['STORAGE_PROVIDER'] if storage_provider == "AZURE": return AzureMissingResourceHttpError( "Missing resource!", 404) if raise_except else AzureMissingResourceHttpError elif storage_provider == "S3": return Exception elif storage_provider == "GCP": return Exception
def test_attach_without_lun(self, mock_datetime): # mock no data disks attached has to result in lun 0 assigned later self.service.get_data_disk.side_effect = AzureMissingResourceHttpError( 'NOT FOUND', 404) mock_datetime.isoformat.return_value = '0' self.service.add_data_disk.return_value = self.my_request result = self.data_disk.attach(self.disk_name, self.cloud_service_name) self.service.add_data_disk.assert_called_once_with( self.cloud_service_name, self.cloud_service_name, self.cloud_service_name, 0, disk_name=self.disk_name)
def test_delete_blobs(self): blob_names = ['to/blob1.txt', 'to/blob2.txt', 'to/blob3.txt'] self.azure_handler.delete_blobs('test', blob_names) self.mock_blob_service.delete_blob.assert_has_calls([ mock.call('test', 'to/blob1.txt'), mock.call('test', 'to/blob2.txt'), mock.call('test', 'to/blob3.txt'), ]) self.mock_azure.reset_mock() self.mock_blob_service.delete_blob = mock.Mock( side_effect=AzureMissingResourceHttpError('A', 'B')) # 判断抛错 six.assertCountEqual( self, self.azure_handler.delete_blobs('test', blob_names), [])
def test_cloud_service(self): mock = Mock() self.service.service.get_hosted_service_properties = mock mock.return_value = True self.assertTrue( self.service.cloud_service_exists(test_conf.meanless_name)) mock.return_value = None self.assertFalse( self.service.cloud_service_exists(test_conf.meanless_name)) mock.side_effect = AzureMissingResourceHttpError(233, 233) self.assertFalse( self.service.cloud_service_exists(test_conf.meanless_name))
def test_list_blobs(self): # Since “name” is an argument to the Mock constructor, # if you want your mock object to have a “name” attribute # you can’t just pass it in at creation time. mock_obj1 = mock.MagicMock() mock_obj1.name = "a.txt" mock_obj2 = mock.MagicMock() mock_obj2.name = "a.wav" mock_obj3 = mock.MagicMock() mock_obj3.name = "a.metadata" self.mock_blob_service.list_blobs.return_value = [ mock_obj1, mock_obj2, mock_obj3 ] six.assertCountEqual(self, self.azure_handler.list_blobs('test'), ["a.txt", "a.wav", "a.metadata"]) self.mock_blob_service.list_blobs.assert_called_with('test', prefix=None, timeout=300) six.assertCountEqual( self, self.azure_handler.list_blobs('test', prefix="test-", suffix=".wav"), ["a.wav"]) self.mock_blob_service.list_blobs.assert_called_with('test', prefix="test-", timeout=300) six.assertCountEqual( self, self.azure_handler.list_blobs('test', suffix=(".wav", ".txt")), ["a.wav", "a.txt"]) self.mock_blob_service.list_blobs.assert_called_with('test', prefix=None, timeout=300) self.mock_azure.reset_mock() self.mock_blob_service.list_blobs = mock.Mock( side_effect=AzureMissingResourceHttpError('a', 'b')) six.assertCountEqual( self, self.azure_handler.list_blobs('test', suffix=(".wav", ".txt")), [])
def get_data_batch_from_blob(result_loc_, prefix=None, backup=False): try: account_name = os.environ['AZURE_STORAGE_ACCOUNT'] account_key = os.environ['AZURE_STORAGE_ACCESS_KEY'] block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) container_name = 'portal-reports-backup' if backup else 'reports' blob_list = block_blob_service.list_blobs(container_name, prefix=prefix) for blob in blob_list: blob_result_loc_ = result_loc_.joinpath(blob.name).parent os.makedirs(blob_result_loc_, exist_ok=True) block_blob_service.get_blob_to_path(container_name=container_name, blob_name=blob.name, file_path=str(result_loc_) + "/" + blob.name) except AzureMissingResourceHttpError: raise AzureMissingResourceHttpError("Missing resource!", 404)
def get_data_from_store(container_name, blob_name, file_path, is_private=False): try: if is_private: account_name = os.environ['AZURE_STORAGE_ACCOUNT'] account_key = os.environ['AZURE_STORAGE_ACCESS_KEY'] else: account_name = os.environ['PUBLIC_AZURE_STORAGE_ACCOUNT'] account_key = os.environ['PUBLIC_AZURE_STORAGE_ACCESS_KEY'] block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) block_blob_service.get_blob_to_path(container_name=container_name, blob_name=blob_name, file_path=file_path) except AzureMissingResourceHttpError: raise AzureMissingResourceHttpError("Missing resource!", 404) except Exception as e: raise Exception('Could not read from blob!' + str(e))
def get_batch_data_from_blob(result_loc_, prefix_, backup=False): try: account_name = os.environ['PUBLIC_AZURE_STORAGE_ACCOUNT'] account_key = os.environ['PUBLIC_AZURE_STORAGE_ACCESS_KEY'] block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) if backup: container_name = os.environ['REPORT_BACKUP_CONTAINER'] blob_list = block_blob_service.list_blobs(container_name, prefix=prefix_) for blob in blob_list: report_name, date_name, file_name = blob.name.split('/') if file_name.startswith('content_consumption_lastweek'): slug = file_name.split('_')[-1].split('.')[0] result_loc_.joinpath(slug).mkdir(exist_ok=True) file_path_ = result_loc_.joinpath( slug, 'content_consumption_lastweek_{}.csv'.format( date_name)) block_blob_service.get_blob_to_path( container_name=container_name, blob_name=blob.name, file_path=str(file_path_)) except AzureMissingResourceHttpError: raise AzureMissingResourceHttpError("Missing resource!", 404)
def get_entity(self, table_name, partition_key, key): full_key = f"{table_name}/{partition_key}/{key}" try: return self.data[full_key] except KeyError: raise AzureMissingResourceHttpError("Not found", 404)
def given_download_exception(self): error = AzureMissingResourceHttpError('injected error', 404) self.email_server_client_mock.download.return_value = ('id', 'folder') self.azure_client_mock.get_blob_to_stream.side_effect = error