def delete(self, obj_id):
super().delete(obj_id) # Check delete permission
if obj_id not in self:
raise ObjNotFoundError(obj_id)
hex_obj_id = hashutil.hash_to_hex(obj_id)
try:
os.remove(self.slicer.get_path(hex_obj_id))
except FileNotFoundError:
raise ObjNotFoundError(obj_id)
return True
async def _get_async(self, obj_id, container_clients=None):
"""Coroutine implementing ``get(obj_id)`` using azure-storage-blob's
asynchronous implementation.
While ``get(obj_id)`` does not need asynchronicity, this is useful to
``get_batch(obj_ids)``, as it can run multiple ``_get_async`` tasks
concurrently."""
if container_clients is None:
# If the container_clients argument is not passed, create a new
# collection of container_clients and restart the function with it.
async with self.get_async_container_clients() as container_clients:
return await self._get_async(obj_id, container_clients)
hex_obj_id = self._internal_id(obj_id)
client = self.get_async_blob_client(hex_obj_id, container_clients)
try:
download = await client.download_blob()
except ResourceNotFoundError:
raise ObjNotFoundError(obj_id) from None
else:
data = await download.content_as_bytes()
decompressor = decompressors[self.compression]()
ret = decompressor.decompress(data)
if decompressor.unused_data:
raise Error("Corrupt object %s: trailing data found" % hex_obj_id)
return ret
def get_stream(self, obj_id, chunk_size=DEFAULT_CHUNK_SIZE):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
data = io.BytesIO(self.state[obj_id])
reader = functools.partial(data.read, chunk_size)
yield from iter(reader, b"")
def delete(self, obj_id):
super().delete(obj_id) # Check delete permission
if obj_id not in self:
raise ObjNotFoundError(obj_id)
self.state.pop(obj_id)
return True
def get(self, obj_id):
for storage in self.get_read_threads(obj_id):
try:
return storage.get(obj_id)
except ObjNotFoundError:
continue
# If no storage contains this content, raise the error
raise ObjNotFoundError(obj_id)
def delete(self, obj_id):
"""Delete an object."""
super().delete(obj_id) # Check delete permission
hex_obj_id = self._internal_id(obj_id)
client = self.get_blob_client(hex_obj_id)
try:
client.delete_blob()
except ResourceNotFoundError:
raise ObjNotFoundError(obj_id) from None
return True
def get(self, obj_id):
try:
obj = self.wf.get(self._path(obj_id))
except Exception:
raise ObjNotFoundError(obj_id)
d = decompressors[self.compression]()
ret = d.decompress(obj)
if d.unused_data:
hex_obj_id = hashutil.hash_to_hex(obj_id)
raise Error("Corrupt object %s: trailing data found" % hex_obj_id)
return ret
def _get_object(self, obj_id):
"""Get a Libcloud wrapper for an object pointer.
This wrapper does not retrieve the content of the object
directly.
"""
object_path = self._object_path(obj_id)
try:
return self.driver.get_object(self.container_name, object_path)
except ObjectDoesNotExistError:
raise ObjNotFoundError(obj_id)
def check(self, obj_id):
nb_present = 0
for storage in self.get_read_threads(obj_id):
try:
storage.check(obj_id)
except ObjNotFoundError:
continue
else:
nb_present += 1
# If there is an Error because of a corrupted file, then let it pass.
# Raise the ObjNotFoundError only if the content couldn't be found in
# all the storages.
if nb_present == 0:
raise ObjNotFoundError(obj_id)
def get_stream(self, obj_id, chunk_size=DEFAULT_CHUNK_SIZE):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
hex_obj_id = hashutil.hash_to_hex(obj_id)
decompressor = decompressors[self.compression]()
with open(self.slicer.get_path(hex_obj_id), "rb") as f:
while True:
raw = f.read(chunk_size)
if not raw:
break
r = decompressor.decompress(raw)
if not r:
continue
yield r
def get(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
# Open the file and return its content as bytes
hex_obj_id = hashutil.hash_to_hex(obj_id)
d = decompressors[self.compression]()
with open(self.slicer.get_path(hex_obj_id), "rb") as f:
out = d.decompress(f.read())
if d.unused_data:
raise Error(
"Corrupt object %s: trailing data found" % hex_obj_id,
)
return out
def delete(self, obj_id):
super().delete(obj_id) # Check delete permission
if obj_id not in self:
raise ObjNotFoundError(obj_id)
self.wf.delete(self._path(obj_id))
return True
def check(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
if compute_hash(self.state[obj_id]) != obj_id:
raise Error("Corrupt object %s" % obj_id)
return True
def get(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
return self.state[obj_id]
def check(self, obj_id, *args, **kwargs):
if self.is_valid(obj_id):
return self.storage.check(*args, obj_id=obj_id, **kwargs)
raise ObjNotFoundError(obj_id)
def get(self, obj_id):
try:
return self.values[obj_id]
except KeyError:
raise ObjNotFoundError(obj_id)
