async def _get_async(self, obj_id, container_clients=None):
"""Coroutine implementing ``get(obj_id)`` using azure-storage-blob's
asynchronous implementation.
While ``get(obj_id)`` does not need asynchronicity, this is useful to
``get_batch(obj_ids)``, as it can run multiple ``_get_async`` tasks
concurrently."""
if container_clients is None:
# If the container_clients argument is not passed, create a new
# collection of container_clients and restart the function with it.
async with self.get_async_container_clients() as container_clients:
return await self._get_async(obj_id, container_clients)
hex_obj_id = self._internal_id(obj_id)
client = self.get_async_blob_client(hex_obj_id, container_clients)
try:
download = await client.download_blob()
except ResourceNotFoundError:
raise ObjNotFoundError(obj_id) from None
else:
data = await download.content_as_bytes()
decompressor = decompressors[self.compression]()
ret = decompressor.decompress(data)
if decompressor.unused_data:
raise Error("Corrupt object %s: trailing data found" % hex_obj_id)
return ret
def check(self, obj_id):
# Check that the file exists, as _get_object raises ObjNotFoundError
self._get_object(obj_id)
# Check the content integrity
obj_content = self.get(obj_id)
content_obj_id = compute_hash(obj_content)
if content_obj_id != obj_id:
raise Error(obj_id)
def get(self, obj_id):
obj = b"".join(self._get_object(obj_id).as_stream())
d = decompressors[self.compression]()
ret = d.decompress(obj)
if d.unused_data:
hex_obj_id = hashutil.hash_to_hex(obj_id)
raise Error("Corrupt object %s: trailing data found" % hex_obj_id)
return ret
def get(self, obj_id):
try:
obj = self.wf.get(self._path(obj_id))
except Exception:
raise ObjNotFoundError(obj_id)
d = decompressors[self.compression]()
ret = d.decompress(obj)
if d.unused_data:
hex_obj_id = hashutil.hash_to_hex(obj_id)
raise Error("Corrupt object %s: trailing data found" % hex_obj_id)
return ret
def check(self, obj_id):
try:
data = self.get(obj_id)
except OSError:
hex_obj_id = hashutil.hash_to_hex(obj_id)
raise Error(
"Corrupt object %s: not a proper compressed file" % hex_obj_id,
)
checksums = hashutil.MultiHash.from_data(
data, hash_names=[ID_HASH_ALGO]
).digest()
actual_obj_id = checksums[ID_HASH_ALGO]
hex_obj_id = hashutil.hash_to_hex(obj_id)
if hex_obj_id != hashutil.hash_to_hex(actual_obj_id):
raise Error(
"Corrupt object %s should have id %s"
% (hashutil.hash_to_hex(obj_id), hashutil.hash_to_hex(actual_obj_id))
)
def get(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
# Open the file and return its content as bytes
hex_obj_id = hashutil.hash_to_hex(obj_id)
d = decompressors[self.compression]()
with open(self.slicer.get_path(hex_obj_id), "rb") as f:
out = d.decompress(f.read())
if d.unused_data:
raise Error(
"Corrupt object %s: trailing data found" % hex_obj_id,
)
return out
def check(self, obj_id):
"""Check the content integrity."""
obj_content = self.get(obj_id)
content_obj_id = compute_hash(obj_content)
if content_obj_id != obj_id:
raise Error(obj_id)
def check(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
if compute_hash(self.state[obj_id]) != obj_id:
raise Error("Corrupt object %s" % obj_id)
return True