def _validate_file_upload(par, file_bucket, file_key, objsize, checksum): """Call this function to signify that the file associated with this PAR has been uploaded. This will check that the objsize and checksum match with what was promised """ from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket from Acquire.Service import get_this_service as _get_this_service service = _get_this_service() bucket = _get_service_account_bucket() file_bucket = _ObjectStore.get_bucket(bucket=bucket, bucket_name=file_bucket, create_if_needed=True) # check that the file uploaded matches what was promised (real_objsize, real_checksum) = _ObjectStore.get_size_and_checksum( file_bucket, file_key) if real_objsize != objsize or real_checksum != checksum: # probably should delete the broken object here... from Acquire.Storage import FileValidationError raise FileValidationError( "The file uploaded does not match what was promised. " "size: %s versus %s, checksum: %s versus %s. Please try " "to upload the file again." % (real_objsize, objsize, real_checksum, checksum))
def _get_metadata_bucket(self): """Return the bucket that contains all of the metadata about the files for this drive """ from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket from Acquire.Service import get_this_service as _get_this_service service = _get_this_service() bucket = _get_service_account_bucket() bucket_name = "user_metadata" try: return _ObjectStore.get_bucket(bucket=bucket, bucket_name=bucket_name, create_if_needed=True) except Exception as e: from Acquire.ObjectStore import RequestBucketError raise RequestBucketError("Unable to open the bucket '%s': %s" % (bucket_name, str(e)))
def _get_file_bucket(self, filekey=None): """Return the bucket that contains the file data for the file associated with 'filekey' in this drive """ from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket from Acquire.Service import get_this_service as _get_this_service service = _get_this_service() bucket = _get_service_account_bucket() bucket_name = self._get_file_bucketname(filekey=filekey) try: return _ObjectStore.get_bucket(bucket=bucket, bucket_name=bucket_name, create_if_needed=True) except Exception as e: from Acquire.ObjectStore import RequestBucketError raise RequestBucketError("Unable to open the bucket '%s': %s" % (bucket_name, str(e)))
def test_objstore(bucket): keys = [] message = "ƒƒƒ Hello World ∂∂∂" ObjectStore.set_string_object(bucket, "test", message) keys.append("test") assert(message == ObjectStore.get_string_object(bucket, "test")) message = "€€#¢∞ Hello ˚ƒ´πµçµΩ" ObjectStore.set_string_object(bucket, "test/something", message) keys.append("test/something") assert(message == ObjectStore.get_string_object(bucket, "test/something")) data = {"cat": "mieow", "dog": "woof", "sounds": [1, 2, 3, 4, 5], "flag": True} ObjectStore.set_object_from_json(bucket, "test/object", data) keys.append("test/object") assert(data == ObjectStore.get_object_from_json(bucket, "test/object")) names = ObjectStore.get_all_object_names(bucket) assert(len(names) == len(keys)) names = ObjectStore.get_all_object_names(bucket, "test") assert(len(names) == 3) names = ObjectStore.get_all_object_names(bucket, "test/") assert(len(names) == 2) names = ObjectStore.get_all_object_names(bucket, "test/some") assert(len(names) == 1) for name in names: assert(name in keys) new_bucket = ObjectStore.create_bucket(bucket, "new_bucket") ObjectStore.set_object_from_json(new_bucket, "test/object2", data) assert(data == ObjectStore.get_object_from_json(new_bucket, "test/object2")) with pytest.raises(ObjectStoreError): new_bucket = ObjectStore.create_bucket(bucket, "testing_objstore") with pytest.raises(ObjectStoreError): new_bucket = ObjectStore.create_bucket(bucket, "new_bucket") with pytest.raises(ObjectStoreError): new_bucket = ObjectStore.get_bucket(bucket, "get_bucket", create_if_needed=False) new_bucket = ObjectStore.get_bucket(bucket, "get_bucket", create_if_needed=True) test_key = "test_string" test_value = "test_string_value" ObjectStore.set_string_object(new_bucket, test_key, test_value) new_bucket2 = ObjectStore.get_bucket(bucket, "get_bucket", create_if_needed=False) test_value2 = ObjectStore.get_string_object(new_bucket2, test_key) assert(test_value == test_value2)