def add(self, content, obj_id=None, check_presence=True):
"""Add an obj in storage if it's not there already."""
if obj_id is None:
# Checksum is missing, compute it on the fly.
obj_id = compute_hash(content)
if check_presence and obj_id in self:
return obj_id
hex_obj_id = self._internal_id(obj_id)
# Send the compressed content
compressor = compressors[self.compression]()
data = compressor.compress(content)
data += compressor.flush()
client = self.get_blob_client(hex_obj_id)
try:
client.upload_blob(data=data, length=len(data))
except ResourceExistsError:
# There's a race condition between check_presence and upload_blob,
# that we can't get rid of as the azure api doesn't allow atomic
# replaces or renaming a blob. As the restore operation explicitly
# removes the blob, it should be safe to just ignore the error.
pass
return obj_id
def check(self, obj_id):
# Check that the file exists, as _get_object raises ObjNotFoundError
self._get_object(obj_id)
# Check the content integrity
obj_content = self.get(obj_id)
content_obj_id = compute_hash(obj_content)
if content_obj_id != obj_id:
raise Error(obj_id)
def restore(self, content, obj_id=None, *args, **kwargs):
if obj_id is None:
obj_id = compute_hash(content)
if self.is_valid(obj_id):
return self.storage.restore(content,
*args,
obj_id=obj_id,
**kwargs)
def restore(self, content, obj_id=None):
"""Restore a content."""
if obj_id is None:
# Checksum is missing, compute it on the fly.
obj_id = compute_hash(content)
if obj_id in self:
self.delete(obj_id)
return self.add(content, obj_id, check_presence=False)
def add(self, content, obj_id=None, check_presence=True):
if obj_id is None:
obj_id = compute_hash(content)
if check_presence and obj_id in self:
return obj_id
self.state[obj_id] = content
return obj_id
def add(self, content, obj_id=None, check_presence=True):
if obj_id is None:
# Checksum is missing, compute it on the fly.
obj_id = compute_hash(content)
if check_presence and obj_id in self:
return obj_id
self._put_object(content, obj_id)
return obj_id
def add(self, content, obj_id=None, check_presence=True):
if obj_id is None:
obj_id = compute_hash(content)
if check_presence and obj_id in self:
# If the object is already present, return immediately.
return obj_id
hex_obj_id = hashutil.hash_to_hex(obj_id)
if not isinstance(content, Iterator):
content = [content]
compressor = compressors[self.compression]()
with self._write_obj_file(hex_obj_id) as f:
for chunk in content:
f.write(compressor.compress(chunk))
f.write(compressor.flush())
return obj_id
def add(self, content, obj_id=None, check_presence=True):
if obj_id is None:
# Checksum is missing, compute it on the fly.
obj_id = compute_hash(content)
if check_presence and obj_id in self:
return obj_id
def compressor(data):
comp = compressors[self.compression]()
for chunk in data:
yield comp.compress(chunk)
yield comp.flush()
if isinstance(content, bytes):
content = [content]
# XXX should handle streaming correctly...
self.wf.put(io.BytesIO(b"".join(compressor(content))),
self._path(obj_id))
return obj_id
def test_list_content(self):
all_ids = []
for i in range(1200):
content = b"example %d" % i
obj_id = compute_hash(content)
self.storage.add(content, obj_id)
all_ids.append(obj_id)
all_ids.sort()
ids = list(self.storage.list_content())
self.assertEqual(len(ids), 1200)
self.assertEqual(ids[0], all_ids[0])
self.assertEqual(ids[100], all_ids[100])
self.assertEqual(ids[999], all_ids[999])
ids = list(self.storage.list_content(limit=10))
self.assertEqual(len(ids), 10)
self.assertEqual(ids[0], all_ids[0])
self.assertEqual(ids[9], all_ids[9])
ids = list(self.storage.list_content(last_obj_id=all_ids[999], limit=100))
self.assertEqual(len(ids), 100)
self.assertEqual(ids[0], all_ids[1000])
self.assertEqual(ids[9], all_ids[1009])
def check(self, obj_id):
"""Check the content integrity."""
obj_content = self.get(obj_id)
content_obj_id = compute_hash(obj_content)
if content_obj_id != obj_id:
raise Error(obj_id)
def _get_internal_id(self, bundle_type, swhid: CoreSWHID):
return compute_hash("{}:{}".format(bundle_type, swhid).encode())
def check(self, obj_id):
# Check the content integrity
obj_content = self.get(obj_id)
content_obj_id = compute_hash(obj_content)
if content_obj_id != obj_id:
raise exc.Error(obj_id)
def check(self, obj_id):
if obj_id not in self:
raise ObjNotFoundError(obj_id)
if compute_hash(self.state[obj_id]) != obj_id:
raise Error("Corrupt object %s" % obj_id)
return True
def hash_content(self, content):
obj_id = compute_hash(content)
return content, obj_id
def add(self, content, obj_id=None, check_presence=True, *args, **kwargs):
if obj_id is None:
obj_id = compute_hash(content)
if self.is_valid(obj_id):
return self.storage.add(content, *args, obj_id=obj_id, **kwargs)
def ensure_invalid(self, content):
obj_id = compute_hash(content)
hex_obj_id = hashutil.hash_to_hex(obj_id)
self.assertFalse(hex_obj_id.startswith(self.prefix))
return content
