def _validate_file_upload(par, file_bucket, file_key, objsize, checksum):
"""Call this function to signify that the file associated with
this PAR has been uploaded. This will check that the
objsize and checksum match with what was promised
"""
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
from Acquire.Service import get_this_service as _get_this_service
service = _get_this_service()
bucket = _get_service_account_bucket()
file_bucket = _ObjectStore.get_bucket(bucket=bucket,
bucket_name=file_bucket,
create_if_needed=True)
# check that the file uploaded matches what was promised
(real_objsize, real_checksum) = _ObjectStore.get_size_and_checksum(
file_bucket, file_key)
if real_objsize != objsize or real_checksum != checksum:
# probably should delete the broken object here...
from Acquire.Storage import FileValidationError
raise FileValidationError(
"The file uploaded does not match what was promised. "
"size: %s versus %s, checksum: %s versus %s. Please try "
"to upload the file again." %
(real_objsize, objsize, real_checksum, checksum))
def _get_metadata_bucket(self):
"""Return the bucket that contains all of the metadata about
the files for this drive
"""
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
from Acquire.Service import get_this_service as _get_this_service
service = _get_this_service()
bucket = _get_service_account_bucket()
bucket_name = "user_metadata"
try:
return _ObjectStore.get_bucket(bucket=bucket,
bucket_name=bucket_name,
create_if_needed=True)
except Exception as e:
from Acquire.ObjectStore import RequestBucketError
raise RequestBucketError("Unable to open the bucket '%s': %s" %
(bucket_name, str(e)))
def _get_file_bucket(self, filekey=None):
"""Return the bucket that contains the file data for the
file associated with 'filekey' in this drive
"""
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
from Acquire.Service import get_this_service as _get_this_service
service = _get_this_service()
bucket = _get_service_account_bucket()
bucket_name = self._get_file_bucketname(filekey=filekey)
try:
return _ObjectStore.get_bucket(bucket=bucket,
bucket_name=bucket_name,
create_if_needed=True)
except Exception as e:
from Acquire.ObjectStore import RequestBucketError
raise RequestBucketError("Unable to open the bucket '%s': %s" %
(bucket_name, str(e)))
def test_objstore(bucket):
keys = []
message = "ƒƒƒ Hello World ∂∂∂"
ObjectStore.set_string_object(bucket, "test", message)
keys.append("test")
assert(message == ObjectStore.get_string_object(bucket, "test"))
message = "€€#¢∞ Hello ˚ƒ´πµçµΩ"
ObjectStore.set_string_object(bucket, "test/something", message)
keys.append("test/something")
assert(message == ObjectStore.get_string_object(bucket, "test/something"))
data = {"cat": "mieow",
"dog": "woof",
"sounds": [1, 2, 3, 4, 5],
"flag": True}
ObjectStore.set_object_from_json(bucket, "test/object", data)
keys.append("test/object")
assert(data == ObjectStore.get_object_from_json(bucket, "test/object"))
names = ObjectStore.get_all_object_names(bucket)
assert(len(names) == len(keys))
names = ObjectStore.get_all_object_names(bucket, "test")
assert(len(names) == 3)
names = ObjectStore.get_all_object_names(bucket, "test/")
assert(len(names) == 2)
names = ObjectStore.get_all_object_names(bucket, "test/some")
assert(len(names) == 1)
for name in names:
assert(name in keys)
new_bucket = ObjectStore.create_bucket(bucket, "new_bucket")
ObjectStore.set_object_from_json(new_bucket, "test/object2", data)
assert(data == ObjectStore.get_object_from_json(new_bucket,
"test/object2"))
with pytest.raises(ObjectStoreError):
new_bucket = ObjectStore.create_bucket(bucket, "testing_objstore")
with pytest.raises(ObjectStoreError):
new_bucket = ObjectStore.create_bucket(bucket, "new_bucket")
with pytest.raises(ObjectStoreError):
new_bucket = ObjectStore.get_bucket(bucket, "get_bucket",
create_if_needed=False)
new_bucket = ObjectStore.get_bucket(bucket, "get_bucket",
create_if_needed=True)
test_key = "test_string"
test_value = "test_string_value"
ObjectStore.set_string_object(new_bucket, test_key, test_value)
new_bucket2 = ObjectStore.get_bucket(bucket, "get_bucket",
create_if_needed=False)
test_value2 = ObjectStore.get_string_object(new_bucket2, test_key)
assert(test_value == test_value2)
