示例#1
0
    def create_par(bucket,
                   encrypt_key,
                   key=None,
                   readable=True,
                   writeable=False,
                   duration=3600,
                   cleanup_function=None):
        """Create a pre-authenticated request for the passed bucket and
           key (if key is None then the request is for the entire bucket).
           This will return a OSPar object that will contain a URL that can
           be used to access the object/bucket. If writeable is true, then
           the URL will also allow the object/bucket to be written to.
           PARs are time-limited. Set the lifetime in seconds by passing
           in 'duration' (by default this is one hour)

           Args:
                bucket (dict): Bucket to create OSPar for
                encrypt_key (PublicKey): Public key to
                encrypt PAR
                key (str, default=None): Key
                readable (bool, default=True): If bucket is readable
                writeable (bool, default=False): If bucket is writeable
                duration (int, default=3600): Duration OSPar should be
                valid for in seconds
                cleanup_function (function, default=None): Cleanup
                function to be passed to PARRegistry

           Returns:
                OSPar: Pre-authenticated request for the bucket
        """
        from Acquire.Crypto import PublicKey as _PublicKey

        if not isinstance(encrypt_key, _PublicKey):
            from Acquire.Client import PARError
            raise PARError("You must supply a valid PublicKey to encrypt the "
                           "returned OSPar")

        is_bucket = (key is None)

        if writeable:
            method = "PUT"
        elif readable:
            method = "GET"
        else:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError("Unsupported permissions model for OSPar!")

        try:
            # get the UTC datetime when this OSPar should expire
            from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
            created_datetime = _get_datetime_now()
            expires_datetime = _get_datetime_now() + _datetime.timedelta(
                seconds=duration)
            bucket_obj = bucket["bucket"]
            if is_bucket:
                url = bucket_obj.generate_signed_url(
                    version='v4', expiration=expires_datetime, method=method)
            else:
                blob = bucket_obj.blob(key)
                url = blob.generate_signed_url(version='v4',
                                               expiration=expires_datetime,
                                               method=method)

        except Exception as e:
            # couldn't create the preauthenticated request
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError("Unable to create the OSPar '%s': %s" %
                                   (key, str(e)))

        if url is None:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError("Unable to create the signed URL!")

        # get the checksum for this URL - used to validate the close
        # request
        from Acquire.ObjectStore import OSPar as _OSPar
        from Acquire.ObjectStore import OSParRegistry as _OSParRegistry
        url_checksum = _OSPar.checksum(url)
        bucket_name = bucket["bucket_name"]
        driver_details = {
            "driver": "gcp",
            "bucket": bucket_name,
            "created_datetime": created_datetime
        }

        par = _OSPar(url=url,
                     encrypt_key=encrypt_key,
                     key=key,
                     expires_datetime=expires_datetime,
                     is_readable=readable,
                     is_writeable=writeable,
                     driver_details=driver_details)

        _OSParRegistry.register(par=par,
                                url_checksum=url_checksum,
                                details_function=_get_driver_details_from_par,
                                cleanup_function=cleanup_function)

        return par
示例#2
0
    def create_par(bucket, encrypt_key, key=None, readable=True,
                   writeable=False, duration=3600, cleanup_function=None):
        """Create a pre-authenticated request for the passed bucket and
           key (if key is None then the request is for the entire bucket).
           This will return a OSPar object that will contain a URL that can
           be used to access the object/bucket. If writeable is true, then
           the URL will also allow the object/bucket to be written to.
           PARs are time-limited. Set the lifetime in seconds by passing
           in 'duration' (by default this is one hour)

           Args:
                bucket (dict): Bucket to create OSPar for
                encrypt_key (PublicKey): Public key to
                encrypt PAR
                key (str, default=None): Key
                readable (bool, default=True): If bucket is readable
                writeable (bool, default=False): If bucket is writeable
                duration (int, default=3600): Duration OSPar should be
                valid for in seconds
                cleanup_function (function, default=None): Cleanup
                function to be passed to PARRegistry

           Returns:
                OSPar: Pre-authenticated request for the bucket
        """
        from Acquire.Crypto import PublicKey as _PublicKey

        if not isinstance(encrypt_key, _PublicKey):
            from Acquire.Client import PARError
            raise PARError(
                "You must supply a valid PublicKey to encrypt the "
                "returned OSPar")

        # get the UTC datetime when this OSPar should expire
        from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
        expires_datetime = _get_datetime_now() + \
            _datetime.timedelta(seconds=duration)

        is_bucket = (key is None)

        # Limitation of OCI - cannot have a bucket OSPar with
        # read permissions!
        if is_bucket and readable:
            from Acquire.Client import PARError
            raise PARError(
                "You cannot create a Bucket OSPar that has read permissions "
                "due to a limitation in the underlying platform")

        try:
            from oci.object_storage.models import \
                CreatePreauthenticatedRequestDetails as \
                _CreatePreauthenticatedRequestDetails
        except:
            raise ImportError(
                "Cannot import OCI. Please install OCI, e.g. via "
                "'pip install oci' so that you can connect to the "
                "Oracle Cloud Infrastructure")

        oci_par = None

        request = _CreatePreauthenticatedRequestDetails()

        if is_bucket:
            request.access_type = "AnyObjectWrite"
        elif readable and writeable:
            request.access_type = "ObjectReadWrite"
        elif readable:
            request.access_type = "ObjectRead"
        elif writeable:
            request.access_type = "ObjectWrite"
        else:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError(
                "Unsupported permissions model for OSPar!")

        request.name = str(_uuid.uuid4())

        if not is_bucket:
            request.object_name = _clean_key(key)

        request.time_expires = expires_datetime

        client = bucket["client"]

        try:
            response = client.create_preauthenticated_request(
                                        client.get_namespace().data,
                                        bucket["bucket_name"],
                                        request)

        except Exception as e:
            # couldn't create the preauthenticated request
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError(
                "Unable to create the OSPar '%s': %s" %
                (str(request), str(e)))

        if response.status != 200:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError(
                "Unable to create the OSPar '%s': Status %s, Error %s" %
                (str(request), response.status, str(response.data)))

        oci_par = response.data

        if oci_par is None:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError(
                "Unable to create the preauthenticated request!")

        created_datetime = oci_par.time_created.replace(
                                tzinfo=_datetime.timezone.utc)

        expires_datetime = oci_par.time_expires.replace(
                                tzinfo=_datetime.timezone.utc)

        # the URI returned by OCI does not include the server. We need
        # to get the server based on the region of this bucket
        url = _get_object_url_for_region(bucket["region"],
                                         oci_par.access_uri)

        # get the checksum for this URL - used to validate the close
        # request
        from Acquire.ObjectStore import OSPar as _OSPar
        from Acquire.ObjectStore import OSParRegistry as _OSParRegistry
        url_checksum = _OSPar.checksum(url)

        driver_details = {"driver": "oci",
                          "bucket": bucket["bucket_name"],
                          "created_datetime": created_datetime,
                          "par_id": oci_par.id,
                          "par_name": oci_par.name}

        par = _OSPar(url=url, encrypt_key=encrypt_key,
                     key=oci_par.object_name,
                     expires_datetime=expires_datetime,
                     is_readable=readable,
                     is_writeable=writeable,
                     driver_details=driver_details)

        _OSParRegistry.register(par=par,
                                url_checksum=url_checksum,
                                details_function=_get_driver_details_from_par,
                                cleanup_function=cleanup_function)

        return par
    def create_par(bucket,
                   encrypt_key,
                   key=None,
                   readable=True,
                   writeable=False,
                   duration=3600,
                   cleanup_function=None):
        """Create a pre-authenticated request for the passed bucket and
           key (if key is None then the request is for the entire bucket).
           This will return a PAR object that will contain a URL that can
           be used to access the object/bucket. If writeable is true, then
           the URL will also allow the object/bucket to be written to.
           PARs are time-limited. Set the lifetime in seconds by passing
           in 'duration' (by default this is one hour). Note that you must
           pass in a public key that will be used to encrypt this PAR. This is
           necessary as the PAR grants access to anyone who can decrypt
           the URL
        """
        from Acquire.Crypto import PublicKey as _PublicKey

        if not isinstance(encrypt_key, _PublicKey):
            from Acquire.Client import PARError
            raise PARError("You must supply a valid PublicKey to encrypt the "
                           "returned PAR")

        if key is not None:
            if not _os.path.exists("%s/%s._data" % (bucket, key)):
                from Acquire.Client import PARError
                raise PARError(
                    "The object '%s' in bucket '%s' does not exist!" %
                    (key, bucket))
        elif not _os.path.exists(bucket):
            from Acquire.Client import PARError
            raise PARError("The bucket '%s' does not exist!" % bucket)

        url = "file://%s" % bucket

        if key:
            url = "%s/%s" % (url, key)

        # get the time this PAR was created
        from Acquire.ObjectStore import get_datetime_now as _get_datetime_now
        created_datetime = _get_datetime_now()

        # get the UTC datetime when this PAR should expire
        expires_datetime = created_datetime + \
            _datetime.timedelta(seconds=duration)

        # mimic limitations of OCI - cannot have a bucket PAR with
        # read permissions!
        if (key is None) and readable:
            from Acquire.Client import PARError
            raise PARError(
                "You cannot create a Bucket PAR that has read permissions "
                "due to a limitation in the underlying platform")

        from Acquire.ObjectStore import OSPar as _OSPar
        from Acquire.ObjectStore import OSParRegistry as _OSParRegistry

        url_checksum = _OSPar.checksum(url)

        driver_details = {
            "driver": "testing_objstore",
            "bucket": bucket,
            "created_datetime": created_datetime
        }

        par = _OSPar(url=url,
                     key=key,
                     encrypt_key=encrypt_key,
                     expires_datetime=expires_datetime,
                     is_readable=readable,
                     is_writeable=writeable,
                     driver_details=driver_details)

        _OSParRegistry.register(par=par,
                                url_checksum=url_checksum,
                                details_function=_get_driver_details_from_par,
                                cleanup_function=cleanup_function)

        return par
示例#4
0
    def download(self, filename=None, version=None,
                 dir=None, force_par=False):
        """Download this file into the local directory
           the local directory, or 'dir' if specified,
           calling the file 'filename' (or whatever it is called
           on the Drive if not specified). If a local
           file exists with this name, then a new, unique filename
           will be used. This returns the local filename of the
           downloaded file (with full absolute path)

           Note that this only downloads files for which you
           have read-access. If the file is not readable then
           an exception is raised and nothing is returned

           If 'version' is specified then download a specific version
           of the file. Otherwise download the version associated
           with this file object
        """
        if self.is_null():
            raise PermissionError("Cannot download a null File!")

        if self._creds is None:
            raise PermissionError("We have not properly opened the file!")

        if filename is None:
            filename = self._metadata.name()

        drive_uid = self._metadata.drive().uid()

        from Acquire.Client import create_new_file as \
            _create_new_file

        if self._creds.is_user():
            privkey = self._creds.user().session_key()
        else:
            from Acquire.Crypto import get_private_key as _get_private_key
            privkey = _get_private_key("parkey")

        args = {"drive_uid": drive_uid,
                "filename": self._metadata.name(),
                "encryption_key": privkey.public_key().to_data()}

        if self._creds.is_user():
            from Acquire.Client import Authorisation as _Authorisation
            authorisation = _Authorisation(
                        resource="download %s %s" % (drive_uid,
                                                     self._metadata.name()),
                        user=self._creds.user())
            args["authorisation"] = authorisation.to_data()
        elif self._creds.is_par():
            par = self._creds.par()
            par.assert_valid()
            args["par_uid"] = par.uid()
            args["secret"] = self._creds.secret()

        if force_par:
            args["force_par"] = True

        if version is not None:
            from Acquire.ObjectStore import datetime_to_string \
                as _datetime_to_string
            args["version"] = _datetime_to_string(version)
        elif self._metadata.version() is not None:
            args["version"] = self._metadata.version()

        storage_service = self._creds.storage_service()

        response = storage_service.call_function(
                                function="download", args=args)

        from Acquire.Client import FileMeta as _FileMeta
        filemeta = _FileMeta.from_data(response["filemeta"])

        if "filedata" in response:
            # we have already downloaded the file to 'filedata'
            filedata = response["filedata"]

            from Acquire.ObjectStore import string_to_bytes \
                as _string_to_bytes
            filedata = _string_to_bytes(response["filedata"])
            del response["filedata"]

            # validate that the size and checksum are correct
            filemeta.assert_correct_data(filedata)

            if filemeta.is_compressed():
                # uncompress the data
                from Acquire.Client import uncompress as _uncompress
                filedata = _uncompress(
                                inputdata=filedata,
                                compression_type=filemeta.compression_type())

            # write the data to the specified local file...
            filename = _create_new_file(filename=filename, dir=dir)
            with open(filename, "wb") as FILE:
                FILE.write(filedata)
                FILE.flush()
        elif "download_par" in response:
            from Acquire.ObjectStore import OSPar as _OSPar
            filename = _create_new_file(filename=filename, dir=dir)
            par = _OSPar.from_data(response["download_par"])
            par.read(privkey).get_object_as_file(filename)
            par.close(privkey)

            # validate that the size and checksum are correct
            filemeta.assert_correct_data(filename=filename)

            # uncompress the file if desired
            if filemeta.is_compressed():
                from Acquire.Client import uncompress as _uncompress
                _uncompress(inputfile=filename,
                            outputfile=filename,
                            compression_type=filemeta.compression_type())

        elif "downloader" in response:
            from Acquire.Client import ChunkDownloader as _ChunkDownloader
            downloader = _ChunkDownloader.from_data(response["downloader"],
                                                    privkey=privkey,
                                                    service=storage_service)

            filename = downloader.download(filename=filename, dir=dir)

        filemeta._copy_credentials(self._metadata)
        self._metadata = filemeta

        return filename
示例#5
0
    def upload(self, filename, force_par=False, aclrules=None):
        """Upload 'filename' as the new version of this file"""
        if self.is_null():
            raise PermissionError("Cannot download a null File!")

        if self._creds is None:
            raise PermissionError("We have not properly opened the file!")

        from Acquire.Client import Authorisation as _Authorisation
        from Acquire.ObjectStore import OSPar as _OSPar
        from Acquire.Client import FileMeta as _FileMeta
        from Acquire.Storage import FileHandle as _FileHandle

        local_cutoff = None

        if force_par:
            # only upload using a OSPar
            local_cutoff = 0

        uploaded_name = self._metadata.filename()
        drive_uid = self._metadata.drive().uid()

        filehandle = _FileHandle(filename=filename,
                                 remote_filename=uploaded_name,
                                 drive_uid=drive_uid,
                                 aclrules=aclrules,
                                 local_cutoff=local_cutoff)

        try:
            args = {"filehandle": filehandle.to_data()}

            if self._creds.is_user():
                authorisation = _Authorisation(
                            resource="upload %s" % filehandle.fingerprint(),
                            user=self._creds.user())

                args["authorisation"] = authorisation.to_data()
            elif self._creds.is_par():
                par = self._creds.par()
                par.assert_valid()
                args["par_uid"] = par.uid()
                args["secret"] = self._creds.secret()
            else:
                raise PermissionError(
                    "Either a logged-in user or valid PAR must be provided!")

            if not filehandle.is_localdata():
                # we will need to upload against a OSPar, so need to tell
                # the service how to encrypt the OSPar...
                if self._creds.is_user():
                    privkey = self._creds.user().session_key()
                else:
                    from Acquire.Crypto import get_private_key \
                        as _get_private_key
                    privkey = _get_private_key("parkey")

                args["encryption_key"] = privkey.public_key().to_data()

            # will eventually need to authorise payment...
            storage_service = self._creds.storage_service()

            response = storage_service.call_function(
                                    function="upload", args=args)

            filemeta = _FileMeta.from_data(response["filemeta"])

            # if this was a large file, then we will receive a OSPar back
            # which must be used to upload the file
            if not filehandle.is_localdata():
                par = _OSPar.from_data(response["upload_par"])
                par.write(privkey).set_object_from_file(
                                        filehandle.local_filename())
                par.close(privkey)

            filemeta._set_drive_metadata(self._metadata._drive_metadata,
                                         self._creds)

            return filemeta
        except:
            # ensure that any temporary files are removed
            filehandle.__del__()
            raise
    def get(par_uid, details_function, url_checksum=None):
        """Return the PAR that matches the passed PAR_UID.
           If 'url_checksum' is supplied then this verifies that
           the checksum of the secret URL is correct.

           This returns the PAR with a completed 'driver_details'.
           The 'driver_details' is created from the dictionary
           of data saved with the PAR. The signature should be;

           driver_details = details_function(data)
        """
        if par_uid is None or len(par_uid) == 0:
            return

        from Acquire.Service import is_running_service as _is_running_service

        if not _is_running_service():
            return

        from Acquire.Service import get_service_account_bucket \
            as _get_service_account_bucket

        from Acquire.ObjectStore import OSPar as _OSPar

        from Acquire.ObjectStore import ObjectStore as _ObjectStore
        from Acquire.ObjectStore import string_to_datetime \
            as _string_to_datetime

        key = "%s/uid/%s" % (_registry_key, par_uid)

        bucket = _get_service_account_bucket()

        objs = _ObjectStore.get_all_objects_from_json(bucket=bucket,
                                                      prefix=key)

        data = None

        for obj in objs.values():
            if url_checksum is not None:
                if url_checksum == obj["url_checksum"]:
                    data = obj
                    break
            else:
                data = obj
                break

        if data is None:
            from Acquire.ObjectStore import ObjectStoreError
            raise ObjectStoreError(
                "There is matching PAR available to close...")

        par = _OSPar.from_data(data["par"])

        if "driver_details" in data:
            if details_function is not None:
                driver_details = details_function(data["driver_details"])
                par._driver_details = driver_details
            else:
                par._driver_details = driver_details

        return par
示例#7
0
def test_par(bucket):
    privkey = get_private_key()
    pubkey = privkey.public_key()

    # first try to create a PAR for the whole bucket
    par = ObjectStore.create_par(bucket, readable=False, writeable=True,
                                 duration=100, encrypt_key=pubkey)

    # should not take 10 seconds to create and return the PAR...
    assert(par.seconds_remaining(buffer=0) > 90)
    assert(par.seconds_remaining(buffer=0) < 101)

    # trying to create a par for a non-existant object should fail
    key = "something"
    value = "∆ƒ^ø  ®∆ ®∆ #®∆… ®#€   €"

    with pytest.raises(PARError):
        par = ObjectStore.create_par(bucket, key=key, encrypt_key=pubkey)

    ObjectStore.set_string_object(bucket, key, value)

    par = ObjectStore.create_par(bucket, key=key, duration=60,
                                 encrypt_key=pubkey)

    assert(par.seconds_remaining(buffer=0) > 55)
    assert(par.seconds_remaining(buffer=0) < 61)

    assert(not par.is_writeable())

    assert(par.key() == key)

    val = par.read(privkey).get_string_object()

    assert(val == value)

    value = "∆˚¬#  #ª ƒ∆ ¬¬¬˚¬∂ß ˚¬ ¬¬¬ßßß"

    with pytest.raises(PARPermissionsError):
        par.write(privkey).set_string_object(value)

    # close the PAR and then assert a closed PAR is null
    par.close()
    assert(par.is_null())

    par = ObjectStore.create_par(bucket, key=key, readable=True,
                                 writeable=True, encrypt_key=pubkey)

    data = par.to_data()
    par2 = OSPar.from_data(data)

    value = "something " + str(uuid.uuid4())

    par2.write(privkey).set_string_object(value)

    val = par.read(privkey).get_string_object()

    assert(val == value)

    par = ObjectStore.create_par(bucket, encrypt_key=pubkey, key=key,
                                 writeable=True, duration=60)

    par.write(privkey).set_string_object(value)

    assert(par.read(privkey).get_string_object() == value)

    assert(ObjectStore.get_string_object(bucket, key) == value)

    par = ObjectStore.create_par(bucket, readable=False,
                                 writeable=True, duration=120,
                                 encrypt_key=pubkey)

    assert(not par.is_readable())
    assert(par.is_writeable())
    assert(par.is_bucket())

    d = "testing"

    keyvals = {"one": "^¬#∆˚¬€", "two": "∆¡πª¨ƒ∆",
               "three": "€√≠ç~ç~€", "four": "hello world!",
               "subdir/five": "#º©√∆˚∆˚¬€ €˚∆ƒ¬"}

    for (key, value) in keyvals.items():
        par.write(privkey).set_string_object("%s/%s" % (d, key), value)

    for key in keyvals.keys():
        par = ObjectStore.create_par(bucket, key="%s/%s" % (d, key),
                                     duration=60, encrypt_key=pubkey)
        value = par.read(privkey).get_string_object()

        assert(keyvals[key] == value)