def upload_chunk(self, file_uid, chunk_index, secret, chunk, checksum):
"""Upload a chunk of the file with UID 'file_uid'. This is the
chunk at index 'chunk_idx', which is set equal to 'chunk'
(validated with 'checksum'). The passed secret is used to
authenticate this upload. The secret should be the
multi_md5 has of the shared secret with the concatenated
drive_uid, file_uid and chunk_index
"""
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.Service import get_service_account_bucket \
as _get_service_account_bucket
bucket = _get_service_account_bucket()
key = "%s/%s/%s" % (_uploader_root, self._drive_uid, file_uid)
data = _ObjectStore.get_object_from_json(bucket, key)
shared_secret = data["secret"]
from Acquire.Crypto import Hash as _Hash
shared_secret = _Hash.multi_md5(
shared_secret, "%s%s%d" % (self._drive_uid, file_uid, chunk_index))
if secret != shared_secret:
raise PermissionError(
"Invalid chunked upload secret. You do not have permission "
"to upload chunks to this file!")
# validate the data checksum
check = _Hash.md5(chunk)
if check != checksum:
from Acquire.Storage import FileValidationError
raise FileValidationError(
"Invalid checksum for chunk: %s versus %s" % (check, checksum))
meta = {
"filesize": len(chunk),
"checksum": checksum,
"compression": "bz2"
}
file_key = data["filekey"]
chunk_index = int(chunk_index)
file_bucket = self._get_file_bucket(file_key)
data_key = "%s/data/%d" % (file_key, chunk_index)
meta_key = "%s/meta/%d" % (file_key, chunk_index)
from Acquire.ObjectStore import ObjectStore as _ObjectStore
_ObjectStore.set_object_from_json(file_bucket, meta_key, meta)
_ObjectStore.set_object(file_bucket, data_key, chunk)
def upload(self,
filehandle,
authorisation=None,
encrypt_key=None,
par=None,
identifiers=None):
"""Upload the file associated with the passed filehandle.
If the filehandle has the data embedded, then this uploads
the file data directly and returns a FileMeta for the
result. Otherwise, this returns a PAR which should
be used to upload the data. The PAR will be encrypted
using 'encrypt_key'. Remember to close the PAR once the
file has been uploaded, so that it can be validated
as correct
"""
from Acquire.Storage import FileHandle as _FileHandle
from Acquire.Storage import FileInfo as _FileInfo
from Acquire.Crypto import PublicKey as _PublicKey
from Acquire.ObjectStore import ObjectStore as _ObjectStore
from Acquire.ObjectStore import string_to_encoded \
as _string_to_encoded
if not isinstance(filehandle, _FileHandle):
raise TypeError("The fileinfo must be of type FileInfo")
if encrypt_key is not None:
if not isinstance(encrypt_key, _PublicKey):
raise TypeError("The encryption key must be of type PublicKey")
(drive_acl, identifiers) = self._resolve_acl(
authorisation=authorisation,
resource="upload %s" % filehandle.fingerprint(),
par=par,
identifiers=identifiers)
if not drive_acl.is_writeable():
raise PermissionError(
"You do not have permission to write to this drive. "
"Your permissions are %s" % str(drive_acl))
# now generate a FileInfo for this FileHandle
fileinfo = _FileInfo(drive_uid=self._drive_uid,
filehandle=filehandle,
identifiers=identifiers,
upstream=drive_acl)
# resolve the ACL for the file from this FileHandle
filemeta = fileinfo.get_filemeta()
file_acl = filemeta.acl()
if not file_acl.is_writeable():
raise PermissionError(
"Despite having write permission to the drive, you "
"do not have write permission for the file. Your file "
"permissions are %s" % str(file_acl))
file_key = fileinfo.latest_version()._file_key()
file_bucket = self._get_file_bucket(file_key)
filedata = None
if filehandle.is_localdata():
# the filehandle already contains the file, so save it
# directly
filedata = filehandle.local_filedata()
_ObjectStore.set_object(bucket=file_bucket,
key=file_key,
data=filedata)
if filedata is None:
# the file is too large to include in the filehandle so
# we need to use a OSPar to upload
from Acquire.ObjectStore import Function as _Function
f = _Function(function=_validate_file_upload,
file_bucket=self._get_file_bucketname(),
file_key=file_key,
objsize=fileinfo.filesize(),
checksum=fileinfo.checksum())
ospar = _ObjectStore.create_par(bucket=file_bucket,
encrypt_key=encrypt_key,
key=file_key,
readable=False,
writeable=True,
cleanup_function=f)
else:
ospar = None
# now save the fileinfo to the object store
fileinfo.save()
filemeta = fileinfo.get_filemeta()
# return the PAR if we need to have a second-stage of upload
return (filemeta, ospar)
