def upload(self,
filehandle,
authorisation=None,
encrypt_key=None,
par=None,
identifiers=None):
"""Upload the file associated with the passed filehandle.
If the filehandle has the data embedded, then this uploads
the file data directly and returns a FileMeta for the
result. Otherwise, this returns a PAR which should
be used to upload the data. The PAR will be encrypted
using 'encrypt_key'. Remember to close the PAR once the
file has been uploaded, so that it can be validated
as correct
"""
from Acquire.Storage import FileHandle as _FileHandle
from Acquire.Storage import FileInfo as _FileInfo
from Acquire.Crypto import PublicKey as _PublicKey
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.ObjectStore import string_to_encoded \
as _string_to_encoded
if not isinstance(filehandle, _FileHandle):
raise TypeError("The fileinfo must be of type FileInfo")
if encrypt_key is not None:
if not isinstance(encrypt_key, _PublicKey):
raise TypeError("The encryption key must be of type PublicKey")
(drive_acl, identifiers) = self._resolve_acl(
authorisation=authorisation,
resource="upload %s" % filehandle.fingerprint(),
par=par,
identifiers=identifiers)
if not drive_acl.is_writeable():
raise PermissionError(
"You do not have permission to write to this drive. "
"Your permissions are %s" % str(drive_acl))
# now generate a FileInfo for this FileHandle
fileinfo = _FileInfo(drive_uid=self._drive_uid,
filehandle=filehandle,
identifiers=identifiers,
upstream=drive_acl)
# resolve the ACL for the file from this FileHandle
filemeta = fileinfo.get_filemeta()
file_acl = filemeta.acl()
if not file_acl.is_writeable():
raise PermissionError(
"Despite having write permission to the drive, you "
"do not have write permission for the file. Your file "
"permissions are %s" % str(file_acl))
file_key = fileinfo.latest_version()._file_key()
file_bucket = self._get_file_bucket(file_key)
filedata = None
if filehandle.is_localdata():
# the filehandle already contains the file, so save it
# directly
filedata = filehandle.local_filedata()
_ObjectStore.set_object(bucket=file_bucket,
key=file_key,
data=filedata)
if filedata is None:
# the file is too large to include in the filehandle so
# we need to use a OSPar to upload
from Acquire.ObjectStore import Function as _Function
f = _Function(function=_validate_file_upload,
file_bucket=self._get_file_bucketname(),
file_key=file_key,
objsize=fileinfo.filesize(),
checksum=fileinfo.checksum())
ospar = _ObjectStore.create_par(bucket=file_bucket,
encrypt_key=encrypt_key,
key=file_key,
readable=False,
writeable=True,
cleanup_function=f)
else:
ospar = None
# now save the fileinfo to the object store
fileinfo.save()
filemeta = fileinfo.get_filemeta()
# return the PAR if we need to have a second-stage of upload
return (filemeta, ospar)
def download(self,
filename,
authorisation,
version=None,
encrypt_key=None,
force_par=False,
must_chunk=False,
par=None,
identifiers=None):
"""Download the file called filename. This will return a
FileHandle that describes the file. If the file is
sufficiently small, then the filedata will be embedded
into this handle. Otherwise a PAR will be generated and
also returned to allow the file to be downloaded
separately. The PAR will be encrypted with 'encrypt_key'.
Remember to close the PAR once you have finished
downloading the file...
"""
from Acquire.Storage import FileHandle as _FileHandle
from Acquire.Storage import FileInfo as _FileInfo
from Acquire.Crypto import PublicKey as _PublicKey
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.ObjectStore import string_to_encoded \
as _string_to_encoded
if not isinstance(encrypt_key, _PublicKey):
raise TypeError("The encryption key must be of type PublicKey")
(drive_acl, identifiers) = self._resolve_acl(
authorisation=authorisation,
resource="download %s %s" % (self._drive_uid, filename),
par=par,
identifiers=identifiers)
# even if the drive_acl is not readable by this user, they
# may have read permission for the file...
# now get the FileInfo for this FileHandle
fileinfo = _FileInfo.load(drive=self,
filename=filename,
version=version,
identifiers=identifiers,
upstream=drive_acl)
# resolve the ACL for the file from this FileHandle
filemeta = fileinfo.get_filemeta()
file_acl = filemeta.acl()
if not file_acl.is_readable():
raise PermissionError(
"You do not have read permissions for the file. Your file "
"permissions are %s" % str(file_acl))
file_bucket = self._get_file_bucket()
file_key = fileinfo.version()._file_key()
filedata = None
downloader = None
ospar = None
if fileinfo.version().is_chunked():
# this is a chunked file. We need to return a
# ChunkDownloader to download the file
from Acquire.Client import ChunkDownloader as _ChunkDownloader
downloader = _ChunkDownloader(drive_uid=self._drive_uid,
file_uid=fileinfo.version().uid())
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
bucket = _get_service_account_bucket()
key = "%s/%s/%s/%s" % (_downloader_root, self._drive_uid,
filemeta.uid(), downloader.uid())
data = {
"filename": filename,
"version": filemeta.uid(),
"filekey": fileinfo.version()._file_key(),
"secret": downloader.secret()
}
_ObjectStore.set_object_from_json(bucket, key, data)
elif must_chunk:
raise PermissionError(
"Cannot download this file in a chunked manner!")
elif force_par or fileinfo.filesize() > 1048576:
# the file is too large to include in the download so
# we need to use a OSPar to download
ospar = _ObjectStore.create_par(bucket=file_bucket,
encrypt_key=encrypt_key,
key=file_key,
readable=True,
writeable=False)
else:
# one-trip download of files that are less than 1 MB
filedata = _ObjectStore.get_object(file_bucket, file_key)
# return the filemeta, and either the filedata, ospar or downloader
return (filemeta, filedata, ospar, downloader)
def test_par(bucket):
privkey = get_private_key()
pubkey = privkey.public_key()
# first try to create a PAR for the whole bucket
par = ObjectStore.create_par(bucket, readable=False, writeable=True,
duration=100, encrypt_key=pubkey)
# should not take 10 seconds to create and return the PAR...
assert(par.seconds_remaining(buffer=0) > 90)
assert(par.seconds_remaining(buffer=0) < 101)
# trying to create a par for a non-existant object should fail
key = "something"
value = "∆ƒ^ø ®∆ ®∆ #®∆… ®#€ €"
with pytest.raises(PARError):
par = ObjectStore.create_par(bucket, key=key, encrypt_key=pubkey)
ObjectStore.set_string_object(bucket, key, value)
par = ObjectStore.create_par(bucket, key=key, duration=60,
encrypt_key=pubkey)
assert(par.seconds_remaining(buffer=0) > 55)
assert(par.seconds_remaining(buffer=0) < 61)
assert(not par.is_writeable())
assert(par.key() == key)
val = par.read(privkey).get_string_object()
assert(val == value)
value = "∆˚¬# #ª ƒ∆ ¬¬¬˚¬∂ß ˚¬ ¬¬¬ßßß"
with pytest.raises(PARPermissionsError):
par.write(privkey).set_string_object(value)
# close the PAR and then assert a closed PAR is null
par.close()
assert(par.is_null())
par = ObjectStore.create_par(bucket, key=key, readable=True,
writeable=True, encrypt_key=pubkey)
data = par.to_data()
par2 = OSPar.from_data(data)
value = "something " + str(uuid.uuid4())
par2.write(privkey).set_string_object(value)
val = par.read(privkey).get_string_object()
assert(val == value)
par = ObjectStore.create_par(bucket, encrypt_key=pubkey, key=key,
writeable=True, duration=60)
par.write(privkey).set_string_object(value)
assert(par.read(privkey).get_string_object() == value)
assert(ObjectStore.get_string_object(bucket, key) == value)
par = ObjectStore.create_par(bucket, readable=False,
writeable=True, duration=120,
encrypt_key=pubkey)
assert(not par.is_readable())
assert(par.is_writeable())
assert(par.is_bucket())
d = "testing"
keyvals = {"one": "^¬#∆˚¬€", "two": "∆¡πª¨ƒ∆",
"three": "€√≠ç~ç~€", "four": "hello world!",
"subdir/five": "#º©√∆˚∆˚¬€ €˚∆ƒ¬"}
for (key, value) in keyvals.items():
par.write(privkey).set_string_object("%s/%s" % (d, key), value)
for key in keyvals.keys():
par = ObjectStore.create_par(bucket, key="%s/%s" % (d, key),
duration=60, encrypt_key=pubkey)
value = par.read(privkey).get_string_object()
assert(keyvals[key] == value)
