def create_par(bucket,
encrypt_key,
key=None,
readable=True,
writeable=False,
duration=3600,
cleanup_function=None):
"""Create a pre-authenticated request for the passed bucket and
key (if key is None then the request is for the entire bucket).
This will return a OSPar object that will contain a URL that can
be used to access the object/bucket. If writeable is true, then
the URL will also allow the object/bucket to be written to.
PARs are time-limited. Set the lifetime in seconds by passing
in 'duration' (by default this is one hour)
Args:
bucket (dict): Bucket to create OSPar for
encrypt_key (PublicKey): Public key to
encrypt PAR
key (str, default=None): Key
readable (bool, default=True): If bucket is readable
writeable (bool, default=False): If bucket is writeable
duration (int, default=3600): Duration OSPar should be
valid for in seconds
cleanup_function (function, default=None): Cleanup
function to be passed to PARRegistry
Returns:
OSPar: Pre-authenticated request for the bucket
"""
from Acquire.Crypto import PublicKey as _PublicKey
if not isinstance(encrypt_key, _PublicKey):
from Acquire.Client import PARError
raise PARError("You must supply a valid PublicKey to encrypt the "
"returned OSPar")
is_bucket = (key is None)
if writeable:
method = "PUT"
elif readable:
method = "GET"
else:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError("Unsupported permissions model for OSPar!")
try:
# get the UTC datetime when this OSPar should expire
from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
created_datetime = _get_datetime_now()
expires_datetime = _get_datetime_now() + _datetime.timedelta(
seconds=duration)
bucket_obj = bucket["bucket"]
if is_bucket:
url = bucket_obj.generate_signed_url(
version='v4', expiration=expires_datetime, method=method)
else:
blob = bucket_obj.blob(key)
url = blob.generate_signed_url(version='v4',
expiration=expires_datetime,
method=method)
except Exception as e:
# couldn't create the preauthenticated request
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError("Unable to create the OSPar '%s': %s" %
(key, str(e)))
if url is None:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError("Unable to create the signed URL!")
# get the checksum for this URL - used to validate the close
# request
from Acquire.ObjectStore import OSPar as _OSPar
from Acquire.ObjectStore import OSParRegistry as _OSParRegistry
url_checksum = _OSPar.checksum(url)
bucket_name = bucket["bucket_name"]
driver_details = {
"driver": "gcp",
"bucket": bucket_name,
"created_datetime": created_datetime
}
par = _OSPar(url=url,
encrypt_key=encrypt_key,
key=key,
expires_datetime=expires_datetime,
is_readable=readable,
is_writeable=writeable,
driver_details=driver_details)
_OSParRegistry.register(par=par,
url_checksum=url_checksum,
details_function=_get_driver_details_from_par,
cleanup_function=cleanup_function)
return par
def create_par(bucket, encrypt_key, key=None, readable=True,
writeable=False, duration=3600, cleanup_function=None):
"""Create a pre-authenticated request for the passed bucket and
key (if key is None then the request is for the entire bucket).
This will return a OSPar object that will contain a URL that can
be used to access the object/bucket. If writeable is true, then
the URL will also allow the object/bucket to be written to.
PARs are time-limited. Set the lifetime in seconds by passing
in 'duration' (by default this is one hour)
Args:
bucket (dict): Bucket to create OSPar for
encrypt_key (PublicKey): Public key to
encrypt PAR
key (str, default=None): Key
readable (bool, default=True): If bucket is readable
writeable (bool, default=False): If bucket is writeable
duration (int, default=3600): Duration OSPar should be
valid for in seconds
cleanup_function (function, default=None): Cleanup
function to be passed to PARRegistry
Returns:
OSPar: Pre-authenticated request for the bucket
"""
from Acquire.Crypto import PublicKey as _PublicKey
if not isinstance(encrypt_key, _PublicKey):
from Acquire.Client import PARError
raise PARError(
"You must supply a valid PublicKey to encrypt the "
"returned OSPar")
# get the UTC datetime when this OSPar should expire
from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
expires_datetime = _get_datetime_now() + \
_datetime.timedelta(seconds=duration)
is_bucket = (key is None)
# Limitation of OCI - cannot have a bucket OSPar with
# read permissions!
if is_bucket and readable:
from Acquire.Client import PARError
raise PARError(
"You cannot create a Bucket OSPar that has read permissions "
"due to a limitation in the underlying platform")
try:
from oci.object_storage.models import \
CreatePreauthenticatedRequestDetails as \
_CreatePreauthenticatedRequestDetails
except:
raise ImportError(
"Cannot import OCI. Please install OCI, e.g. via "
"'pip install oci' so that you can connect to the "
"Oracle Cloud Infrastructure")
oci_par = None
request = _CreatePreauthenticatedRequestDetails()
if is_bucket:
request.access_type = "AnyObjectWrite"
elif readable and writeable:
request.access_type = "ObjectReadWrite"
elif readable:
request.access_type = "ObjectRead"
elif writeable:
request.access_type = "ObjectWrite"
else:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError(
"Unsupported permissions model for OSPar!")
request.name = str(_uuid.uuid4())
if not is_bucket:
request.object_name = _clean_key(key)
request.time_expires = expires_datetime
client = bucket["client"]
try:
response = client.create_preauthenticated_request(
client.get_namespace().data,
bucket["bucket_name"],
request)
except Exception as e:
# couldn't create the preauthenticated request
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError(
"Unable to create the OSPar '%s': %s" %
(str(request), str(e)))
if response.status != 200:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError(
"Unable to create the OSPar '%s': Status %s, Error %s" %
(str(request), response.status, str(response.data)))
oci_par = response.data
if oci_par is None:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError(
"Unable to create the preauthenticated request!")
created_datetime = oci_par.time_created.replace(
tzinfo=_datetime.timezone.utc)
expires_datetime = oci_par.time_expires.replace(
tzinfo=_datetime.timezone.utc)
# the URI returned by OCI does not include the server. We need
# to get the server based on the region of this bucket
url = _get_object_url_for_region(bucket["region"],
oci_par.access_uri)
# get the checksum for this URL - used to validate the close
# request
from Acquire.ObjectStore import OSPar as _OSPar
from Acquire.ObjectStore import OSParRegistry as _OSParRegistry
url_checksum = _OSPar.checksum(url)
driver_details = {"driver": "oci",
"bucket": bucket["bucket_name"],
"created_datetime": created_datetime,
"par_id": oci_par.id,
"par_name": oci_par.name}
par = _OSPar(url=url, encrypt_key=encrypt_key,
key=oci_par.object_name,
expires_datetime=expires_datetime,
is_readable=readable,
is_writeable=writeable,
driver_details=driver_details)
_OSParRegistry.register(par=par,
url_checksum=url_checksum,
details_function=_get_driver_details_from_par,
cleanup_function=cleanup_function)
return par
def create_par(bucket,
encrypt_key,
key=None,
readable=True,
writeable=False,
duration=3600,
cleanup_function=None):
"""Create a pre-authenticated request for the passed bucket and
key (if key is None then the request is for the entire bucket).
This will return a PAR object that will contain a URL that can
be used to access the object/bucket. If writeable is true, then
the URL will also allow the object/bucket to be written to.
PARs are time-limited. Set the lifetime in seconds by passing
in 'duration' (by default this is one hour). Note that you must
pass in a public key that will be used to encrypt this PAR. This is
necessary as the PAR grants access to anyone who can decrypt
the URL
"""
from Acquire.Crypto import PublicKey as _PublicKey
if not isinstance(encrypt_key, _PublicKey):
from Acquire.Client import PARError
raise PARError("You must supply a valid PublicKey to encrypt the "
"returned PAR")
if key is not None:
if not _os.path.exists("%s/%s._data" % (bucket, key)):
from Acquire.Client import PARError
raise PARError(
"The object '%s' in bucket '%s' does not exist!" %
(key, bucket))
elif not _os.path.exists(bucket):
from Acquire.Client import PARError
raise PARError("The bucket '%s' does not exist!" % bucket)
url = "file://%s" % bucket
if key:
url = "%s/%s" % (url, key)
# get the time this PAR was created
from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
created_datetime = _get_datetime_now()
# get the UTC datetime when this PAR should expire
expires_datetime = created_datetime + \
_datetime.timedelta(seconds=duration)
# mimic limitations of OCI - cannot have a bucket PAR with
# read permissions!
if (key is None) and readable:
from Acquire.Client import PARError
raise PARError(
"You cannot create a Bucket PAR that has read permissions "
"due to a limitation in the underlying platform")
from Acquire.ObjectStore import OSPar as _OSPar
from Acquire.ObjectStore import OSParRegistry as _OSParRegistry
url_checksum = _OSPar.checksum(url)
driver_details = {
"driver": "testing_objstore",
"bucket": bucket,
"created_datetime": created_datetime
}
par = _OSPar(url=url,
key=key,
encrypt_key=encrypt_key,
expires_datetime=expires_datetime,
is_readable=readable,
is_writeable=writeable,
driver_details=driver_details)
_OSParRegistry.register(par=par,
url_checksum=url_checksum,
details_function=_get_driver_details_from_par,
cleanup_function=cleanup_function)
return par
def download(self, filename=None, version=None,
dir=None, force_par=False):
"""Download this file into the local directory
the local directory, or 'dir' if specified,
calling the file 'filename' (or whatever it is called
on the Drive if not specified). If a local
file exists with this name, then a new, unique filename
will be used. This returns the local filename of the
downloaded file (with full absolute path)
Note that this only downloads files for which you
have read-access. If the file is not readable then
an exception is raised and nothing is returned
If 'version' is specified then download a specific version
of the file. Otherwise download the version associated
with this file object
"""
if self.is_null():
raise PermissionError("Cannot download a null File!")
if self._creds is None:
raise PermissionError("We have not properly opened the file!")
if filename is None:
filename = self._metadata.name()
drive_uid = self._metadata.drive().uid()
from Acquire.Client import create_new_file as \
_create_new_file
if self._creds.is_user():
privkey = self._creds.user().session_key()
else:
from Acquire.Crypto import get_private_key as _get_private_key
privkey = _get_private_key("parkey")
args = {"drive_uid": drive_uid,
"filename": self._metadata.name(),
"encryption_key": privkey.public_key().to_data()}
if self._creds.is_user():
from Acquire.Client import Authorisation as _Authorisation
authorisation = _Authorisation(
resource="download %s %s" % (drive_uid,
self._metadata.name()),
user=self._creds.user())
args["authorisation"] = authorisation.to_data()
elif self._creds.is_par():
par = self._creds.par()
par.assert_valid()
args["par_uid"] = par.uid()
args["secret"] = self._creds.secret()
if force_par:
args["force_par"] = True
if version is not None:
from Acquire.ObjectStore import datetime_to_string \
as _datetime_to_string
args["version"] = _datetime_to_string(version)
elif self._metadata.version() is not None:
args["version"] = self._metadata.version()
storage_service = self._creds.storage_service()
response = storage_service.call_function(
function="download", args=args)
from Acquire.Client import FileMeta as _FileMeta
filemeta = _FileMeta.from_data(response["filemeta"])
if "filedata" in response:
# we have already downloaded the file to 'filedata'
filedata = response["filedata"]
from Acquire.ObjectStore import string_to_bytes \
as _string_to_bytes
filedata = _string_to_bytes(response["filedata"])
del response["filedata"]
# validate that the size and checksum are correct
filemeta.assert_correct_data(filedata)
if filemeta.is_compressed():
# uncompress the data
from Acquire.Client import uncompress as _uncompress
filedata = _uncompress(
inputdata=filedata,
compression_type=filemeta.compression_type())
# write the data to the specified local file...
filename = _create_new_file(filename=filename, dir=dir)
with open(filename, "wb") as FILE:
FILE.write(filedata)
FILE.flush()
elif "download_par" in response:
from Acquire.ObjectStore import OSPar as _OSPar
filename = _create_new_file(filename=filename, dir=dir)
par = _OSPar.from_data(response["download_par"])
par.read(privkey).get_object_as_file(filename)
par.close(privkey)
# validate that the size and checksum are correct
filemeta.assert_correct_data(filename=filename)
# uncompress the file if desired
if filemeta.is_compressed():
from Acquire.Client import uncompress as _uncompress
_uncompress(inputfile=filename,
outputfile=filename,
compression_type=filemeta.compression_type())
elif "downloader" in response:
from Acquire.Client import ChunkDownloader as _ChunkDownloader
downloader = _ChunkDownloader.from_data(response["downloader"],
privkey=privkey,
service=storage_service)
filename = downloader.download(filename=filename, dir=dir)
filemeta._copy_credentials(self._metadata)
self._metadata = filemeta
return filename
def upload(self, filename, force_par=False, aclrules=None):
"""Upload 'filename' as the new version of this file"""
if self.is_null():
raise PermissionError("Cannot download a null File!")
if self._creds is None:
raise PermissionError("We have not properly opened the file!")
from Acquire.Client import Authorisation as _Authorisation
from Acquire.ObjectStore import OSPar as _OSPar
from Acquire.Client import FileMeta as _FileMeta
from Acquire.Storage import FileHandle as _FileHandle
local_cutoff = None
if force_par:
# only upload using a OSPar
local_cutoff = 0
uploaded_name = self._metadata.filename()
drive_uid = self._metadata.drive().uid()
filehandle = _FileHandle(filename=filename,
remote_filename=uploaded_name,
drive_uid=drive_uid,
aclrules=aclrules,
local_cutoff=local_cutoff)
try:
args = {"filehandle": filehandle.to_data()}
if self._creds.is_user():
authorisation = _Authorisation(
resource="upload %s" % filehandle.fingerprint(),
user=self._creds.user())
args["authorisation"] = authorisation.to_data()
elif self._creds.is_par():
par = self._creds.par()
par.assert_valid()
args["par_uid"] = par.uid()
args["secret"] = self._creds.secret()
else:
raise PermissionError(
"Either a logged-in user or valid PAR must be provided!")
if not filehandle.is_localdata():
# we will need to upload against a OSPar, so need to tell
# the service how to encrypt the OSPar...
if self._creds.is_user():
privkey = self._creds.user().session_key()
else:
from Acquire.Crypto import get_private_key \
as _get_private_key
privkey = _get_private_key("parkey")
args["encryption_key"] = privkey.public_key().to_data()
# will eventually need to authorise payment...
storage_service = self._creds.storage_service()
response = storage_service.call_function(
function="upload", args=args)
filemeta = _FileMeta.from_data(response["filemeta"])
# if this was a large file, then we will receive a OSPar back
# which must be used to upload the file
if not filehandle.is_localdata():
par = _OSPar.from_data(response["upload_par"])
par.write(privkey).set_object_from_file(
filehandle.local_filename())
par.close(privkey)
filemeta._set_drive_metadata(self._metadata._drive_metadata,
self._creds)
return filemeta
except:
# ensure that any temporary files are removed
filehandle.__del__()
raise
def get(par_uid, details_function, url_checksum=None):
"""Return the PAR that matches the passed PAR_UID.
If 'url_checksum' is supplied then this verifies that
the checksum of the secret URL is correct.
This returns the PAR with a completed 'driver_details'.
The 'driver_details' is created from the dictionary
of data saved with the PAR. The signature should be;
driver_details = details_function(data)
"""
if par_uid is None or len(par_uid) == 0:
return
from Acquire.Service import is_running_service as _is_running_service
if not _is_running_service():
return
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
from Acquire.ObjectStore import OSPar as _OSPar
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.ObjectStore import string_to_datetime \
as _string_to_datetime
key = "%s/uid/%s" % (_registry_key, par_uid)
bucket = _get_service_account_bucket()
objs = _ObjectStore.get_all_objects_from_json(bucket=bucket,
prefix=key)
data = None
for obj in objs.values():
if url_checksum is not None:
if url_checksum == obj["url_checksum"]:
data = obj
break
else:
data = obj
break
if data is None:
from Acquire.ObjectStore import ObjectStoreError
raise ObjectStoreError(
"There is matching PAR available to close...")
par = _OSPar.from_data(data["par"])
if "driver_details" in data:
if details_function is not None:
driver_details = details_function(data["driver_details"])
par._driver_details = driver_details
else:
par._driver_details = driver_details
return par
def test_par(bucket):
privkey = get_private_key()
pubkey = privkey.public_key()
# first try to create a PAR for the whole bucket
par = ObjectStore.create_par(bucket, readable=False, writeable=True,
duration=100, encrypt_key=pubkey)
# should not take 10 seconds to create and return the PAR...
assert(par.seconds_remaining(buffer=0) > 90)
assert(par.seconds_remaining(buffer=0) < 101)
# trying to create a par for a non-existant object should fail
key = "something"
value = "∆ƒ^ø ®∆ ®∆ #®∆… ®#€ €"
with pytest.raises(PARError):
par = ObjectStore.create_par(bucket, key=key, encrypt_key=pubkey)
ObjectStore.set_string_object(bucket, key, value)
par = ObjectStore.create_par(bucket, key=key, duration=60,
encrypt_key=pubkey)
assert(par.seconds_remaining(buffer=0) > 55)
assert(par.seconds_remaining(buffer=0) < 61)
assert(not par.is_writeable())
assert(par.key() == key)
val = par.read(privkey).get_string_object()
assert(val == value)
value = "∆˚¬# #ª ƒ∆ ¬¬¬˚¬∂ß ˚¬ ¬¬¬ßßß"
with pytest.raises(PARPermissionsError):
par.write(privkey).set_string_object(value)
# close the PAR and then assert a closed PAR is null
par.close()
assert(par.is_null())
par = ObjectStore.create_par(bucket, key=key, readable=True,
writeable=True, encrypt_key=pubkey)
data = par.to_data()
par2 = OSPar.from_data(data)
value = "something " + str(uuid.uuid4())
par2.write(privkey).set_string_object(value)
val = par.read(privkey).get_string_object()
assert(val == value)
par = ObjectStore.create_par(bucket, encrypt_key=pubkey, key=key,
writeable=True, duration=60)
par.write(privkey).set_string_object(value)
assert(par.read(privkey).get_string_object() == value)
assert(ObjectStore.get_string_object(bucket, key) == value)
par = ObjectStore.create_par(bucket, readable=False,
writeable=True, duration=120,
encrypt_key=pubkey)
assert(not par.is_readable())
assert(par.is_writeable())
assert(par.is_bucket())
d = "testing"
keyvals = {"one": "^¬#∆˚¬€", "two": "∆¡πª¨ƒ∆",
"three": "€√≠ç~ç~€", "four": "hello world!",
"subdir/five": "#º©√∆˚∆˚¬€ €˚∆ƒ¬"}
for (key, value) in keyvals.items():
par.write(privkey).set_string_object("%s/%s" % (d, key), value)
for key in keyvals.keys():
par = ObjectStore.create_par(bucket, key="%s/%s" % (d, key),
duration=60, encrypt_key=pubkey)
value = par.read(privkey).get_string_object()
assert(keyvals[key] == value)