def _flush(self, buffer, *_):
"""
Flush the write buffer of the stream if applicable.
Args:
buffer (memoryview): Buffer content.
"""
buffer_size = len(buffer)
# If buffer too large, must flush by parts sequentially
if buffer_size > self.MAX_FLUSH_SIZE:
for part_start in range(0, buffer_size, self.MAX_FLUSH_SIZE):
# Split buffer and append
buffer_part = buffer[part_start:part_start +
self.MAX_FLUSH_SIZE]
with _handle_azure_exception():
self._client.append_block(block=buffer_part.tobytes(),
**self._client_kwargs)
# Small buffer, send it in one command.
elif buffer_size:
with _handle_azure_exception():
self._client.append_block(block=buffer.tobytes(),
**self._client_kwargs)
def _create(self):
"""
Create the file if not exists.
"""
with _handle_azure_exception():
self._client.create_blob_from_bytes(blob=b'',
**self._client_kwargs)
def _list_objects(self, client_kwargs, path, max_request_entries):
"""
Lists objects.
args:
client_kwargs (dict): Client arguments.
path (str): Path relative to current locator.
max_request_entries (int): If specified, maximum entries returned
by request.
Returns:
generator of tuple: object name str, object header dict
"""
client_kwargs = self._update_listing_client_kwargs(
client_kwargs, max_request_entries)
blob = None
with _handle_azure_exception():
for blob in self._client_block.list_blobs(prefix=path,
**client_kwargs):
yield blob.name, self._model_to_dict(blob)
# None only if path don't exists
if blob is None:
raise ObjectNotFoundError
def _list_locators(self):
"""
Lists locators.
Returns:
generator of tuple: locator name str, locator header dict
"""
with _handle_azure_exception():
for container in self._client_block.list_containers():
yield container.name, self._model_to_dict(container)
def _list_locators(self):
"""
Lists locators.
Returns:
generator of tuple: locator name str, locator header dict
"""
with _handle_azure_exception():
for share in self.client.list_shares():
yield share.name, self._model_to_dict(share)
def _flush(self, buffer):
"""
Flush the write buffer of the stream if applicable.
Args:
buffer (memoryview): Buffer content.
"""
with _handle_azure_exception():
# Write entire file at once
self._client.create_blob_from_bytes(blob=buffer.tobytes(),
**self._client_kwargs)
def test_handle_azure_exception():
"""Test pycosio.storage.azure._handle_azure_exception"""
from pycosio.storage.azure import _handle_azure_exception
from azure.common import AzureHttpError
from pycosio._core.exceptions import (ObjectNotFoundError,
ObjectPermissionError)
# Any error
with pytest.raises(AzureHttpError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=400)
# 404 error
with pytest.raises(ObjectNotFoundError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=404)
# 403 error
with pytest.raises(ObjectPermissionError):
with _handle_azure_exception():
raise AzureHttpError(message='', status_code=403)
def _remove(self, client_kwargs):
"""
Remove an object.
args:
client_kwargs (dict): Client arguments.
"""
with _handle_azure_exception():
# Blob
if 'blob_name' in client_kwargs:
return self._client_block.delete_blob(**client_kwargs)
# Container
return self._client_block.delete_container(**client_kwargs)
def copy(self, src, dst, other_system=None):
"""
Copy object of the same storage.
Args:
src (str): Path or URL.
dst (str): Path or URL.
other_system (pycosio.storage.azure._AzureBaseSystem subclass):
The source storage system.
"""
with _handle_azure_exception():
self._client_block.copy_blob(copy_source=(other_system
or self)._format_src_url(
src, self),
**self.get_client_kwargs(dst))
def _make_dir(self, client_kwargs):
"""
Make a directory.
args:
client_kwargs (dict): Client arguments.
"""
with _handle_azure_exception():
# Blob
if 'blob_name' in client_kwargs:
return self._client_block.create_blob_from_bytes(
blob=b'', **client_kwargs)
# Container
return self._client_block.create_container(**client_kwargs)
def _make_dir(self, client_kwargs):
"""
Make a directory.
args:
client_kwargs (dict): Client arguments.
"""
with _handle_azure_exception():
# Directory
if 'directory_name' in client_kwargs:
return self.client.create_directory(
share_name=client_kwargs['share_name'],
directory_name=client_kwargs['directory_name'])
# Share
return self.client.create_share(**client_kwargs)
def _list_objects(self, client_kwargs, max_request_entries):
"""
Lists objects.
args:
client_kwargs (dict): Client arguments.
max_request_entries (int): If specified, maximum entries returned
by request.
Returns:
generator of tuple: object name str, object header dict,
directory bool
"""
client_kwargs = self._update_listing_client_kwargs(
client_kwargs, max_request_entries)
with _handle_azure_exception():
for obj in self.client.list_directories_and_files(**client_kwargs):
yield (obj.name, self._model_to_dict(obj),
isinstance(obj, _Directory))
def _head(self, client_kwargs):
"""
Returns object or bucket HTTP header.
Args:
client_kwargs (dict): Client arguments.
Returns:
dict: HTTP header.
"""
with _handle_azure_exception():
# Blob
if 'blob_name' in client_kwargs:
result = self._client_block.get_blob_properties(
**client_kwargs)
# Container
else:
result = self._client_block.get_container_properties(
**client_kwargs)
return self._model_to_dict(result)
def _remove(self, client_kwargs):
"""
Remove an object.
args:
client_kwargs (dict): Client arguments.
"""
with _handle_azure_exception():
# File
if 'file_name' in client_kwargs:
return self.client.delete_file(
share_name=client_kwargs['share_name'],
directory_name=client_kwargs['directory_name'],
file_name=client_kwargs['file_name'])
# Directory
elif 'directory_name' in client_kwargs:
return self.client.delete_directory(
share_name=client_kwargs['share_name'],
directory_name=client_kwargs['directory_name'])
# Share
return self.client.delete_share(
share_name=client_kwargs['share_name'])
def _head(self, client_kwargs):
"""
Returns object or bucket HTTP header.
Args:
client_kwargs (dict): Client arguments.
Returns:
dict: HTTP header.
"""
with _handle_azure_exception():
# File
if 'file_name' in client_kwargs:
result = self.client.get_file_properties(**client_kwargs)
# Directory
elif 'directory_name' in client_kwargs:
result = self.client.get_directory_properties(**client_kwargs)
# Share
else:
result = self.client.get_share_properties(**client_kwargs)
return self._model_to_dict(result)
