Example #1
0
def crds(authenticated_user):
    get_local_bucket(empty=True)
    creds = StorageCreds(user=authenticated_user, service_url="storage")
    drive = Drive(creds=creds, name="test_drive")
    filepath = os.path.join(
        os.path.dirname(__file__),
        "../../../tests/data/proc_test_data/CRDS/bsd.picarro.1minute.248m.dat",
    )
    filemeta = drive.upload(filepath)

    par = PAR(location=filemeta.location(), user=authenticated_user)

    hugs = Service(service_url="hugs")
    par_secret = hugs.encrypt_data(par.secret())

    auth = Authorisation(resource="process", user=authenticated_user)

    args = {
        "authorisation": auth.to_data(),
        "par": {
            "data": par.to_data()
        },
        "par_secret": {
            "data": par_secret
        },
        "data_type": "CRDS",
        "source_name": "bsd.picarro.1minute.248m",
    }

    hugs.call_function(function="process", args=args)
Example #2
0
def test_process_CRDS(authenticated_user, tempdir):
    creds = StorageCreds(user=authenticated_user, service_url="storage")
    drive = Drive(creds=creds, name="test_drive")
    filepath = os.path.join(
        os.path.dirname(__file__),
        "../../../tests/data/proc_test_data/CRDS/bsd.picarro.1minute.248m.dat",
    )
    filemeta = drive.upload(filepath)

    Path("/tmp/bsd.picarro.1minute.248m.dat").unlink(missing_ok=True)

    par = PAR(location=filemeta.location(), user=authenticated_user)

    hugs = Service(service_url="hugs")
    par_secret = hugs.encrypt_data(par.secret())

    auth = Authorisation(resource="process", user=authenticated_user)

    args = {
        "authorisation": auth.to_data(),
        "par": {
            "data": par.to_data()
        },
        "par_secret": {
            "data": par_secret
        },
        "data_type": "CRDS",
        "source_name": "bsd.picarro.1minute.248m",
    }

    response = hugs.call_function(function="process", args=args)

    expected_keys = [
        "bsd.picarro.1minute.248m_ch4",
        "bsd.picarro.1minute.248m_co",
        "bsd.picarro.1minute.248m_co2",
    ]

    results = response["results"]["bsd.picarro.1minute.248m.dat"]

    return False

    assert sorted(results.keys()) == expected_keys
def test_chunking(authenticated_user, tempdir):
    drive_name = "test_chunking"
    creds = StorageCreds(user=authenticated_user, service_url="storage")

    drive = Drive(name=drive_name, creds=creds)

    uploader = drive.chunk_upload("test_chunking.py")

    uploader.upload("This is some text\n")
    uploader.upload("Here is")
    uploader.upload(" some more!\n")

    uploader.close()

    filename = drive.download("test_chunking.py", dir=tempdir)

    lines = open(filename).readlines()

    assert(lines[0] == "This is some text\n")
    assert(lines[1] == "Here is some more!\n")

    downloader = drive.chunk_download("test_chunking.py", dir=tempdir)

    filename = downloader.local_filename()

    nchunks = 0
    while downloader.download_next_chunk():
        nchunks = nchunks + 1
        if nchunks > 4:
            assert(False)

    assert(nchunks == 4)  # extra chunk is empty to close

    lines = open(filename).readlines()

    assert(lines[0] == "This is some text\n")
    assert(lines[1] == "Here is some more!\n")
Example #4
0
def test_process_GC(authenticated_user, tempdir):
    creds = StorageCreds(user=authenticated_user, service_url="storage")
    drive = Drive(creds=creds, name="test_drive")
    data_filepath = os.path.join(
        os.path.dirname(__file__),
        "../../../tests/data/proc_test_data/GC/capegrim-medusa.18.C",
    )
    precision_filepath = os.path.join(
        os.path.dirname(__file__),
        "../../../tests/data/proc_test_data/GC/capegrim-medusa.18.precisions.C",
    )

    Path("/tmp/capegrim-medusa.18.C").unlink(missing_ok=True)
    Path("/tmp/capegrim-medusa.18.precisions.C").unlink(missing_ok=True)

    data_meta = drive.upload(data_filepath)
    precision_meta = drive.upload(precision_filepath)

    data_par = PAR(location=data_meta.location(), user=authenticated_user)
    precision_par = PAR(location=precision_meta.location(),
                        user=authenticated_user)

    hugs = Service(service_url="hugs")
    data_secret = hugs.encrypt_data(data_par.secret())
    precision_secret = hugs.encrypt_data(precision_par.secret())

    auth = Authorisation(resource="process", user=authenticated_user)

    args = {
        "authorisation": auth.to_data(),
        "par": {
            "data": data_par.to_data(),
            "precision": precision_par.to_data()
        },
        "par_secret": {
            "data": data_secret,
            "precision": precision_secret
        },
        "data_type": "GCWERKS",
        "source_name": "capegrim-medusa",
        "site": "CGO",
        "instrument": "medusa",
    }

    response = hugs.call_function(function="process", args=args)

    result_keys = (sorted(
        response["results"]["capegrim-medusa.18.C"].keys()))[:8]

    expected_keys = [
        "capegrim-medusa.18_C4F10",
        "capegrim-medusa.18_C6F14",
        "capegrim-medusa.18_CCl4",
        "capegrim-medusa.18_CF4",
        "capegrim-medusa.18_CFC-11",
        "capegrim-medusa.18_CFC-112",
        "capegrim-medusa.18_CFC-113",
        "capegrim-medusa.18_CFC-114",
    ]

    assert result_keys == expected_keys
Example #5
0
    def create_job(
        self,
        auth_user,
        requirements,
        key_password,
        data_files,
        hugs_url=None,
        storage_url=None,
    ):
        """ Create a job

            Args:
                auth_user (Acquire.User): Authenticated Acquire user

                The following keys are required:
                    "hostname", "username", "name", "run_command", "partition", "n_nodes", "n_tasks_per_node",
                    "n_cpus_per_task", "memory_req", "job_duration"
                where partition must be one of:
                    "cpu_test", "dcv", "gpu", "gpu_veryshort", "hmem", "serial", "test", "veryshort"

                Example:
                    requirements = {"hostname": hostname, "username": username, "name": "test_job,
                                    "n_nodes": 2,"n_tasks_per_node": 2,
                                    "n_cpus_per_task": 2, "memory": "128G", ...}

                requirements (dict): Dictionary containing job details and requested resources
                key_password (str): Password for private key used to access the HPC
                data_files (dict): Data file(s) to be uploaded to the cloud drive to
                run the simulation. Simulation code files should be given in the "app" key and data
                files in the "data" key

                TODO - having to pass in a password and get it through to Paramiko seems
                long winded, is there a better way to do this?

                hugs_url (str): URL of HUGS service
                storage_url (str): URL of storage service
            Returns:
                dict: Dictionary containing information regarding job running on resource
                This will contain the PAR for access for data upload and download.
        """
        from Acquire.Client import (
            Drive,
            Service,
            PAR,
            Authorisation,
            StorageCreds,
            Location,
            ACLRule,
        )
        from Acquire.ObjectStore import create_uuid
        import datetime
        import os

        if self._service is None:
            raise PermissionError("Cannot use a null service")

        if storage_url is None:
            storage_url = self._service_url + "/storage"

        if hugs_url is None:
            hugs_url = self._service_url + "/hugs"

        if not isinstance(data_files["app"], list):
            data_files["app"] = [data_files["app"]]

        try:
            if not isinstance(data_files["data"], list):
                data_files["data"] = [data_files["data"]]
        except KeyError:
            pass

        # Get an authorisaton to pass to the service
        hugs = Service(service_url=hugs_url)
        # Credentials to create the cloud storage drive
        creds = StorageCreds(user=auth_user, service_url=storage_url)

        # Append a shortened UUID to the job name to ensure we don't multiple drives with the same name
        short_uuid = create_uuid(short_uid=True)

        job_name = requirements["name"]
        job_name = f"{job_name.lower()}_{short_uuid}"

        # Create a cloud drive for the input and output data to be written to
        drive = Drive(creds=creds, name=job_name)

        # Check the size of the files and if we want to use the chunk uploader
        # Now we want to upload the files to the cloud drive we've created for this job
        chunk_limit = 50 * 1024 * 1024

        # Store the metadata for the uploaded files
        uploaded_files = {"app": {}, "data": {}}
        # These probably won't be very big so don't check their size
        for f in data_files["app"]:
            file_meta = drive.upload(f, dir="app")
            uploaded_files["app"][f] = file_meta

        # We might not have any data files to upload
        try:
            for f in data_files["data"]:
                filesize = os.path.getsize(f)

                if filesize < chunk_limit:
                    file_meta = drive.upload(f, dir="data")
                else:
                    file_meta = drive.chunk_upload(f, dir="data")

                uploaded_files["data"][f] = file_meta
        except KeyError:
            pass

        auth = Authorisation(resource="job_runner", user=auth_user)
        # Create a PAR with a long lifetime here and return a version to the user
        # and another to the server to allow writing of result data
        drive_guid = drive.metadata().guid()
        location = Location(drive_guid=drive_guid)

        # Read the duration from the requirements dictionary

        # TODO - add in some reading of the duration
        # try:
        #     duration = requirements["duration"]
        #     par_expiry = datetime.datetime

        par_lifetime = datetime.datetime.now() + datetime.timedelta(days=1)

        # Create an ACL rule for this PAR so we can read and write to it
        aclrule = ACLRule.owner()
        par = PAR(
            location=location,
            user=auth_user,
            aclrule=aclrule,
            expires_datetime=par_lifetime,
        )

        par_secret = par.secret()
        encryped_par_secret = hugs.encrypt_data(par_secret)

        # Encrypt the password we use to decrypt the private key used to access the HPC cluster
        # TODO - is this a sensible way of doing this?
        encrypted_password = hugs.encrypt_data(key_password)

        par_data = par.to_data()

        args = {}
        args["authorisation"] = auth.to_data()
        args["par"] = par_data
        args["par_secret"] = encryped_par_secret
        args["requirements"] = requirements
        args["key_password"] = encrypted_password

        function_response = self._service.call_function(function="job_runner",
                                                        args=args)

        response = {}
        response["function_response"] = function_response
        response["par"] = par_data
        response["par_secret"] = par_secret
        response["upload_data"] = uploaded_files

        return response
def test_drive_par(authenticated_user, tempdir):
    drive_name = "test å∫ç∂ pars"
    creds = StorageCreds(user=authenticated_user, service_url="storage")

    drive = Drive(name=drive_name, creds=creds)

    drive.upload(filename=__file__, uploaded_name="tmp_test.py")

    downloaded_name = drive.download(filename="tmp_test.py", dir=tempdir)

    assert (_same_file(__file__, downloaded_name))

    drive_guid = drive.metadata().guid()

    location = Location(drive_guid=drive_guid)

    par = PAR(location=location,
              user=authenticated_user,
              aclrule=ACLRule.reader())

    par_drive = par.resolve()
    assert (par_drive.metadata().acl() == ACLRule.reader())
    assert (par_drive.metadata().uid() == drive.metadata().uid())

    files = par_drive.list_files()
    assert (len(files) == 1)
    assert (files[0].filename() == "tmp_test.py")

    downloaded_name = files[0].open().download(dir=tempdir, force_par=True)

    assert (_same_file(__file__, downloaded_name))

    par2 = PAR(location=location,
               user=authenticated_user,
               aclrule=ACLRule.writer())

    par_drive = par2.resolve()

    assert (par_drive.metadata().acl() == ACLRule.writer())
    assert (par_drive.metadata().uid() == drive.metadata().uid())

    files = par_drive.list_files()
    assert (len(files) == 1)
    assert (files[0].filename() == "tmp_test.py")

    par_drive.upload(filename=__file__, uploaded_name="tmp_test2.py")

    files = par_drive.list_files()
    assert (len(files) == 2)
    f = {}
    f[files[0].filename()] = files[0]
    f[files[1].filename()] = files[1]
    files = f

    assert ("tmp_test.py" in files)
    assert ("tmp_test2.py" in files)

    downloaded_name = files["tmp_test2.py"].open().download(dir=tempdir)

    assert (_same_file(__file__, downloaded_name))

    par = PAR(location=files["tmp_test.py"].location(),
              user=authenticated_user,
              aclrule=ACLRule.reader())

    par_file = par.resolve()

    assert (par_file.metadata().acl() == ACLRule.reader())

    downloaded_name = par_file.download(dir=tempdir)

    assert (_same_file(__file__, downloaded_name))

    with pytest.raises(PermissionError):
        par_file.upload(__file__)

    par = PAR(location=files["tmp_test.py"].location(),
              user=authenticated_user,
              aclrule=ACLRule.writer())

    par_file = par.resolve()

    assert (par_file.metadata().acl() == ACLRule.writer())

    par_file.upload(__file__)
Example #7
0
    def process_files(
        self,
        user,
        files,
        data_type,
        source_name=None,
        overwrite=False,
        hugs_url=None,
        storage_url=None,
        datasource=None,
        site=None,
        instrument=None,
    ):
        """ Process the passed file(s)

            Args:
                user (User): Authenticated Acquire User
                files (str, list): Path of files to be processed
                data_type (str): Type of data to be processed (CRDS, GC etc)
                hugs_url (str): URL of HUGS service. Currently used for testing
                datasource (str): Datasource name or UUID
                This may be removed in the future.
                storage_url (str): URL of storage service. Currently used for testing
                This may be removed in the future.
                site (str, default=None): Name of site, three letter code or long name
                instrument (str, default=None): If no instrument name is passed we will attempt
                to find it from the filename.
            Returns:
                dict: UUIDs of Datasources storing data of processed files keyed by filename
        """
        data_type = data_type.upper()

        if self._service is None:
            raise PermissionError("Cannot use a null service")

        if not isinstance(files, list):
            files = [files]

        if data_type.upper() == "GC":
            if not all(isinstance(item, tuple) for item in files):
                return TypeError(
                    "If data type is GC, a list of tuples for data and precision filenames must be passed"
                )

            files = [(Path(f), Path(p)) for f, p in files]
        else:
            files = [Path(f) for f in files]

        if storage_url is None:
            storage_url = self._service_url + "/storage"

        if hugs_url is None:
            hugs_url = self._service_url + "/hugs"

        # # Take the filename without the file extension
        # source_name = [os.path.splitext((filepath.name).split("/")[-1])[0] for filepath in files]

        hugs = Service(service_url=hugs_url)
        creds = StorageCreds(user=user, service_url=storage_url)
        drive = Drive(creds=creds, name="test_drive")
        auth = Authorisation(resource="process", user=user)

        # Here we'll need special cases for different data types. As GC requires
        # both the data file and precision data and they need to be kept together
        # for use in processing.
        # We can maybe reconsider the way this is done if there ends up being a lot of test
        # cases and this gets a bit clunky
        results = {}
        for file in files:
            if data_type == "GC":
                if source_name is None:
                    source_name = file[0].stem

                if site is None:
                    site = source_name.split(".")[0]
                    if "-" in site and data_type == "GC":
                        site = site.split("-")[0]

                filemeta = drive.upload(file[0])
                par = PAR(location=filemeta.location(), user=user)
                par_secret = hugs.encrypt_data(par.secret())

                prec_meta = drive.upload(file[1])
                prec_par = PAR(location=prec_meta.location(), user=user)
                prec_par_secret = hugs.encrypt_data(prec_par.secret())

                args = {
                    "authorisation": auth.to_data(),
                    "par": {"data": par.to_data(), "precision": prec_par.to_data()},
                    "par_secret": {"data": par_secret, "precision": prec_par_secret},
                    "data_type": data_type,
                    "datasource": datasource,
                    "source_name": source_name,
                    "overwrite": overwrite,
                    "site": site,
                    "instrument": instrument,
                }
            else:
                filemeta = drive.upload(file)
                par = PAR(location=filemeta.location(), user=user)
                par_secret = hugs.encrypt_data(par.secret())

                args = {
                    "authorisation": auth.to_data(),
                    "par": {"data": par.to_data()},
                    "par_secret": {"data": par_secret},
                    "data_type": data_type,
                    "datasource": datasource,
                    "source_name": source_name,
                    "overwrite": overwrite,
                }

            # If we try to upload many files we don't want it to fail if a single
            # file contains overlapping data
            try:
                response = self._service.call_function(function="process", args=args)
                results.update(response["results"])
            except ValueError as err:
                results[file.name] = err

        return results
Example #8
0
def test_drives(authenticated_user, tempdir):

    creds = StorageCreds(user=authenticated_user, service_url="storage")

    nstart = len(Drive.list_toplevel_drives(creds=creds))

    drive_name = "test å∫ç∂ something"
    drive = Drive(name=drive_name, creds=creds, autocreate=True)

    assert (drive.metadata().name() == drive_name)
    assert (drive.metadata().acl().is_owner())

    drive2_name = "test/this/is/a/../../dir"

    drive2 = Drive(name=drive2_name, creds=creds)

    drives = Drive.list_toplevel_drives(creds=creds)

    assert (len(drives) == nstart + 2)

    drives = drive2.list_drives()

    assert (len(drives) == 0)

    drives = drive.list_drives()

    assert (len(drives) == 0)

    filename = __file__

    files = drive.list_files()
    assert (len(files) == 0)

    filemeta = drive.upload(filename=filename)

    assert (filemeta.is_complete())
    assert (filemeta.acl().is_owner())
    assert (filemeta.acl().is_readable())
    assert (filemeta.acl().is_writeable())
    assert (filemeta.uploaded_by() == authenticated_user.guid())
    assert (filemeta.uploaded_when() is not None)

    upload_datetime = filemeta.uploaded_when()

    (_, filename) = os.path.split(filename)

    assert (filemeta.filename() == filename)

    files = drive.list_files()

    assert (len(files) == 1)

    assert (files[0].filename() == filemeta.filename())
    assert (not files[0].is_complete())

    files = drive.list_files(include_metadata=True)

    assert (len(files) == 1)

    assert (files[0].filename() == filemeta.filename())
    assert (files[0].is_complete())

    assert (files[0].uid() == filemeta.uid())
    assert (files[0].filesize() == filemeta.filesize())
    assert (files[0].checksum() == filemeta.checksum())
    assert (files[0].compression_type() == filemeta.compression_type())
    assert (files[0].uploaded_by() == filemeta.uploaded_by())
    assert (files[0].uploaded_when() == filemeta.uploaded_when())
    assert (files[0].acl().is_owner())
    assert (files[0].uploaded_by() == authenticated_user.guid())
    assert (files[0].uploaded_when() == upload_datetime)

    f = files[0].open()
    filename = f.download(dir=tempdir)

    # make sure that the two files are identical
    with open(filename, "rb") as FILE:
        data1 = FILE.read()

    # remove this tmp file
    os.unlink(filename)

    with open(__file__, "rb") as FILE:
        data2 = FILE.read()

    assert (data1 == data2)

    assert (files[0].uid() == filemeta.uid())
    assert (files[0].filesize() == filemeta.filesize())
    assert (files[0].checksum() == filemeta.checksum())
    assert (files[0].compression_type() == filemeta.compression_type())
    assert (files[0].uploaded_by() == filemeta.uploaded_by())
    assert (files[0].uploaded_when() == filemeta.uploaded_when())
    assert (files[0].acl().is_owner())
    assert (files[0].uploaded_by() == authenticated_user.guid())
    assert (files[0].uploaded_when() == upload_datetime)

    versions = f.list_versions()

    assert (len(versions) == 1)
    assert (versions[0].filename() == filemeta.filename())
    assert (versions[0].uploaded_when() == filemeta.uploaded_when())

    new_filemeta = drive.upload(filename=__file__, force_par=True)

    versions = f.list_versions()

    assert (len(versions) == 2)

    filename = new_filemeta.open().download(dir=tempdir)

    # make sure that the two files are identical
    with open(filename, "rb") as FILE:
        data1 = FILE.read()

    # remove this tmp file
    os.unlink(filename)

    with open(__file__, "rb") as FILE:
        data2 = FILE.read()

    assert (data1 == data2)

    # should be in upload order
    assert (versions[0].uid() == filemeta.uid())
    assert (versions[1].uid() == new_filemeta.uid())

    filename = new_filemeta.open().download(dir=tempdir, force_par=True)

    # make sure that the two files are identical
    with open(filename, "rb") as FILE:
        data1 = FILE.read()

    # remove this tmp file
    os.unlink(filename)

    assert (data1 == data2)

    # try to upload a file with path to the drive
    filemeta = drive.upload(filename=__file__,
                            uploaded_name="/test/one/../two/test.py")

    assert (filemeta.filename() == "test/two/test.py")

    # cannot create a new Drive with non-owner ACLs
    with pytest.raises(PermissionError):
        drive = Drive(name="broken_acl",
                      creds=creds,
                      aclrules=ACLRules.owner("12345@z0-z0"))

    drive = Drive(name="working_acl",
                  creds=creds,
                  aclrules=ACLRules.owner(authenticated_user.guid()))
def test_run_calc(aaai_services, authenticated_user):
    # create and register the cluster on which this job will take place...
    cluster = Cluster.create(service_url="compute",
                             user=aaai_services["compute"]["user"])

    user = authenticated_user
    assert (user.is_logged_in())

    # ensure that the main account has money
    deposit(user,
            100.0,
            "Adding money to the account",
            accounting_url="accounting")

    # get a handle to the financial account used to pay for the job
    account = Account(user=user,
                      account_name="deposits",
                      accounting_url="accounting")

    assert (account.balance() >= 100.0)

    # Upload a directory that will contain all of the input
    creds = StorageCreds(user=user, service_url="storage")
    drive = Drive(name="sim", creds=creds, autocreate=True)

    uploaded = drive.upload(_testdata())
    location = uploaded.location()

    print(drive.list_files(dir="example_sim/input"))
    print(location)

    # create a request for a job to be run using:
    #  1. 'image_name' as the specified container image for the software
    #  2. 'location' as the location containing all input files

    # authorise it using the authenticated user (who may be different to the
    # user who pays for the job - hence the need for a different
    # authorisation for the request and for the cheque)
    r = RunRequest(image="docker://test_image:latest", input=location)

    # now write a cheque which will provide authorisation to spend money from
    # this account to pay for this request. This will be written to the access
    # service to give it the authority to create a transation in the account.
    # This cheque authorises only a single transaction, performable only
    # by the service whose canonical URL is supplied, and the access service
    # should check that the requested resource signature matches that
    # authorised by the cheque
    cheque = Cheque.write(account=account,
                          recipient_url="access",
                          resource=r.fingerprint(),
                          max_spend=50.0)

    func = "run_calculation"
    args = {}
    args["request"] = r.to_data()
    args["authorisation"] = Authorisation(user=user,
                                          resource=r.fingerprint()).to_data()
    args["cheque"] = cheque.to_data()

    access_service = Service("access")

    result = access_service.call_function(func, args)
    print(result)

    pending_uids = cluster.get_pending_job_uids()

    print(pending_uids)

    for uid in pending_uids:
        job = cluster.submit_job(uid)
        print(job)

    pending_uids = cluster.get_pending_job_uids()

    print(pending_uids)