def upload(self, filehandle, authorisation=None, encrypt_key=None, par=None, identifiers=None): """Upload the file associated with the passed filehandle. If the filehandle has the data embedded, then this uploads the file data directly and returns a FileMeta for the result. Otherwise, this returns a PAR which should be used to upload the data. The PAR will be encrypted using 'encrypt_key'. Remember to close the PAR once the file has been uploaded, so that it can be validated as correct """ from Acquire.Storage import FileHandle as _FileHandle from Acquire.Storage import FileInfo as _FileInfo from Acquire.Crypto import PublicKey as _PublicKey from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.ObjectStore import string_to_encoded \ as _string_to_encoded if not isinstance(filehandle, _FileHandle): raise TypeError("The fileinfo must be of type FileInfo") if encrypt_key is not None: if not isinstance(encrypt_key, _PublicKey): raise TypeError("The encryption key must be of type PublicKey") (drive_acl, identifiers) = self._resolve_acl( authorisation=authorisation, resource="upload %s" % filehandle.fingerprint(), par=par, identifiers=identifiers) if not drive_acl.is_writeable(): raise PermissionError( "You do not have permission to write to this drive. " "Your permissions are %s" % str(drive_acl)) # now generate a FileInfo for this FileHandle fileinfo = _FileInfo(drive_uid=self._drive_uid, filehandle=filehandle, identifiers=identifiers, upstream=drive_acl) # resolve the ACL for the file from this FileHandle filemeta = fileinfo.get_filemeta() file_acl = filemeta.acl() if not file_acl.is_writeable(): raise PermissionError( "Despite having write permission to the drive, you " "do not have write permission for the file. Your file " "permissions are %s" % str(file_acl)) file_key = fileinfo.latest_version()._file_key() file_bucket = self._get_file_bucket(file_key) filedata = None if filehandle.is_localdata(): # the filehandle already contains the file, so save it # directly filedata = filehandle.local_filedata() _ObjectStore.set_object(bucket=file_bucket, key=file_key, data=filedata) if filedata is None: # the file is too large to include in the filehandle so # we need to use a OSPar to upload from Acquire.ObjectStore import Function as _Function f = _Function(function=_validate_file_upload, file_bucket=self._get_file_bucketname(), file_key=file_key, objsize=fileinfo.filesize(), checksum=fileinfo.checksum()) ospar = _ObjectStore.create_par(bucket=file_bucket, encrypt_key=encrypt_key, key=file_key, readable=False, writeable=True, cleanup_function=f) else: ospar = None # now save the fileinfo to the object store fileinfo.save() filemeta = fileinfo.get_filemeta() # return the PAR if we need to have a second-stage of upload return (filemeta, ospar)
def download(self, filename, authorisation, version=None, encrypt_key=None, force_par=False, must_chunk=False, par=None, identifiers=None): """Download the file called filename. This will return a FileHandle that describes the file. If the file is sufficiently small, then the filedata will be embedded into this handle. Otherwise a PAR will be generated and also returned to allow the file to be downloaded separately. The PAR will be encrypted with 'encrypt_key'. Remember to close the PAR once you have finished downloading the file... """ from Acquire.Storage import FileHandle as _FileHandle from Acquire.Storage import FileInfo as _FileInfo from Acquire.Crypto import PublicKey as _PublicKey from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.ObjectStore import string_to_encoded \ as _string_to_encoded if not isinstance(encrypt_key, _PublicKey): raise TypeError("The encryption key must be of type PublicKey") (drive_acl, identifiers) = self._resolve_acl( authorisation=authorisation, resource="download %s %s" % (self._drive_uid, filename), par=par, identifiers=identifiers) # even if the drive_acl is not readable by this user, they # may have read permission for the file... # now get the FileInfo for this FileHandle fileinfo = _FileInfo.load(drive=self, filename=filename, version=version, identifiers=identifiers, upstream=drive_acl) # resolve the ACL for the file from this FileHandle filemeta = fileinfo.get_filemeta() file_acl = filemeta.acl() if not file_acl.is_readable(): raise PermissionError( "You do not have read permissions for the file. Your file " "permissions are %s" % str(file_acl)) file_bucket = self._get_file_bucket() file_key = fileinfo.version()._file_key() filedata = None downloader = None ospar = None if fileinfo.version().is_chunked(): # this is a chunked file. We need to return a # ChunkDownloader to download the file from Acquire.Client import ChunkDownloader as _ChunkDownloader downloader = _ChunkDownloader(drive_uid=self._drive_uid, file_uid=fileinfo.version().uid()) from Acquire.ObjectStore import ObjectStore as _ObjectStore from Acquire.Service import get_service_account_bucket \ as _get_service_account_bucket bucket = _get_service_account_bucket() key = "%s/%s/%s/%s" % (_downloader_root, self._drive_uid, filemeta.uid(), downloader.uid()) data = { "filename": filename, "version": filemeta.uid(), "filekey": fileinfo.version()._file_key(), "secret": downloader.secret() } _ObjectStore.set_object_from_json(bucket, key, data) elif must_chunk: raise PermissionError( "Cannot download this file in a chunked manner!") elif force_par or fileinfo.filesize() > 1048576: # the file is too large to include in the download so # we need to use a OSPar to download ospar = _ObjectStore.create_par(bucket=file_bucket, encrypt_key=encrypt_key, key=file_key, readable=True, writeable=False) else: # one-trip download of files that are less than 1 MB filedata = _ObjectStore.get_object(file_bucket, file_key) # return the filemeta, and either the filedata, ospar or downloader return (filemeta, filedata, ospar, downloader)
def test_par(bucket): privkey = get_private_key() pubkey = privkey.public_key() # first try to create a PAR for the whole bucket par = ObjectStore.create_par(bucket, readable=False, writeable=True, duration=100, encrypt_key=pubkey) # should not take 10 seconds to create and return the PAR... assert(par.seconds_remaining(buffer=0) > 90) assert(par.seconds_remaining(buffer=0) < 101) # trying to create a par for a non-existant object should fail key = "something" value = "∆ƒ^ø ®∆ ®∆ #®∆… ®#€ €" with pytest.raises(PARError): par = ObjectStore.create_par(bucket, key=key, encrypt_key=pubkey) ObjectStore.set_string_object(bucket, key, value) par = ObjectStore.create_par(bucket, key=key, duration=60, encrypt_key=pubkey) assert(par.seconds_remaining(buffer=0) > 55) assert(par.seconds_remaining(buffer=0) < 61) assert(not par.is_writeable()) assert(par.key() == key) val = par.read(privkey).get_string_object() assert(val == value) value = "∆˚¬# #ª ƒ∆ ¬¬¬˚¬∂ß ˚¬ ¬¬¬ßßß" with pytest.raises(PARPermissionsError): par.write(privkey).set_string_object(value) # close the PAR and then assert a closed PAR is null par.close() assert(par.is_null()) par = ObjectStore.create_par(bucket, key=key, readable=True, writeable=True, encrypt_key=pubkey) data = par.to_data() par2 = OSPar.from_data(data) value = "something " + str(uuid.uuid4()) par2.write(privkey).set_string_object(value) val = par.read(privkey).get_string_object() assert(val == value) par = ObjectStore.create_par(bucket, encrypt_key=pubkey, key=key, writeable=True, duration=60) par.write(privkey).set_string_object(value) assert(par.read(privkey).get_string_object() == value) assert(ObjectStore.get_string_object(bucket, key) == value) par = ObjectStore.create_par(bucket, readable=False, writeable=True, duration=120, encrypt_key=pubkey) assert(not par.is_readable()) assert(par.is_writeable()) assert(par.is_bucket()) d = "testing" keyvals = {"one": "^¬#∆˚¬€", "two": "∆¡πª¨ƒ∆", "three": "€√≠ç~ç~€", "four": "hello world!", "subdir/five": "#º©√∆˚∆˚¬€ €˚∆ƒ¬"} for (key, value) in keyvals.items(): par.write(privkey).set_string_object("%s/%s" % (d, key), value) for key in keyvals.keys(): par = ObjectStore.create_par(bucket, key="%s/%s" % (d, key), duration=60, encrypt_key=pubkey) value = par.read(privkey).get_string_object() assert(keyvals[key] == value)