def run(args):
"""This file is used to return a list of versions of the
specified filenames
"""
drive_uid = str(args["drive_uid"])
authorisation = Authorisation.from_data(args["authorisation"])
filename = args["filename"]
try:
include_metadata = args["include_metadata"]
except:
include_metadata = False
if include_metadata:
include_metadata = True
else:
include_metadata = False
drive = DriveInfo(drive_uid=drive_uid)
versions = drive.list_versions(authorisation=authorisation,
filename=filename,
include_metadata=include_metadata)
return_value = {}
return_value["versions"] = list_to_string(versions)
return return_value
def process(args):
""" Take a PAR from an uploaded file and process the data
Args:
args (dict): Dictionary of JSON serialised objects to be
used by processing functions
Returns:
dict: Dictionary of results of processing
"""
data_type = args["data_type"]
data_par = PAR.from_data(args["par"]["data"])
data_secret = args["par_secret"]["data"]
auth = args["authorisation"]
authorisation = Authorisation.from_data(auth)
# Verify that this process had authorisation to be called
authorisation.verify("process")
hugs = get_this_service(need_private_access=True)
data_secret = hugs.decrypt_data(data_secret)
data_filename = data_par.resolve(secret=data_secret)
# Here we're downloading the data to the tmp directory
# Be good if we could load it directly from the object store
data_file = data_filename.download(dir="/tmp")
if data_type == "GC":
precision_par = PAR.from_data(args["par"]["precision"])
precision_secret = args["par_secret"]["precision"]
precision_secret = hugs.decrypt_data(precision_secret)
precision_filename = precision_par.resolve(precision_secret)
precision_file = precision_filename.download(dir="/tmp")
site = args["site"]
instrument = args["instrument"]
data_file = data_file, precision_file
else:
site = None
instrument = None
if "overwrite" in args:
overwrite = args["overwrite"]
else:
overwrite = False
source_name = args["source_name"]
results = process_data(
data_file=data_file,
source_name=source_name,
data_type=data_type,
site=site,
instrument_name=instrument,
overwrite=overwrite,
)
return {"results": results}
def run(args):
"""Open and return a new ChunkUploader that can be used
to upload a file in lots of chunks
"""
filename = str(args["filename"])
drive_uid = str(args["drive_uid"])
try:
aclrules = ACLRules.from_data(args["aclrules"])
except:
aclrules = None
try:
authorisation = Authorisation.from_data(args["authorisation"])
except:
authorisation = None
try:
par_uid = str(args["par_uid"])
except:
par_uid = None
try:
secret = str(args["secret"])
except:
secret = None
try:
pubkey = PublicKey.from_data(args["encryption_key"])
except:
pubkey = None
if par_uid is not None:
registry = PARRegistry()
(par, identifiers) = registry.load(par_uid=par_uid, secret=secret)
else:
par = None
identifiers = None
drive = DriveInfo(drive_uid=drive_uid)
(filemeta, uploader) = drive.open_uploader(filename=filename,
aclrules=aclrules,
authorisation=authorisation,
par=par,
identifiers=identifiers)
return {
"filemeta": filemeta.to_data(),
"uploader": uploader.to_data(pubkey=pubkey)
}
def run(args):
"""Create a new PAR based on the information supplied
in the passed half-created PAR, and the supplied
Authorisation. This will return the URL that will
need to be used by the PAR to access the required
data. This will be encrypted using the supplied
PublicKey
"""
auth = Authorisation.from_data(args["authorisation"])
par = PAR.from_data(args["par"])
secret = args["secret"]
registry = PARRegistry()
par_uid = registry.register(par=par, authorisation=auth, secret=secret)
result = {"par_uid": par_uid}
return result
def job_runner(args):
""" Service function that gets called by the Client job_runner
Args:
dict: Dictionary of variables used in setting up and running job
Returns:
dict: Dictionary of data detailing job run status such as stdout, stderr output
"""
auth = args["authorisation"]
authorisation = Authorisation.from_data(auth)
# Verify that this process had authorisation to be called
authorisation.verify("job_runner")
hugs = get_this_service(need_private_access=True)
job_data = args["requirements"]
# # Pass the PAR through to allow use in the control script
job_data["par"] = args["par"]
# Pass the decrypted PAR secret here as we're on the server already
job_data["par_secret"] = hugs.decrypt_data(args["par_secret"])
hostname = job_data["hostname"]
username = job_data["username"]
# Have we used this server before?
try:
known_host = job_data["known_host"]
except KeyError:
known_host = False
# Decrypt the password we use to access the private key
password = hugs.decrypt_data(args["key_password"])
results = run_job(
username=username,
hostname=hostname,
password=password,
job_data=job_data,
known_host=known_host,
)
return results
def run(args):
"""Admin function used to set the cluster that will be used to
actually perform jobs
"""
try:
authorisation = Authorisation.from_data(args["authorisation"])
except:
authorisation = None
try:
passphrase = str(args["passphrase"])
except:
passphrase = None
cluster = Cluster.from_data(args["cluster"])
Cluster.set_cluster(cluster=cluster,
authorisation=authorisation,
passphrase=passphrase)
return {"cluster": cluster.to_data()}
def run(args):
"""This file is used to return a list of the FileMeta objects of files
that are in a specified drive
"""
drive_uid = str(args["drive_uid"])
try:
authorisation = Authorisation.from_data(args["authorisation"])
except:
authorisation = None
try:
par_uid = args["par_uid"]
except:
par_uid = None
try:
secret = args["secret"]
except:
secret = None
try:
directory = str(args["dir"])
except:
directory = None
try:
filename = str(args["filename"])
except:
filename = None
try:
include_metadata = args["include_metadata"]
except:
include_metadata = False
if include_metadata:
include_metadata = True
else:
include_metadata = False
if par_uid is not None:
registry = PARRegistry()
(par, identifiers) = registry.load(par_uid=par_uid, secret=secret)
else:
par = None
identifiers = None
drive = DriveInfo(drive_uid=drive_uid)
files = drive.list_files(authorisation=authorisation,
include_metadata=include_metadata,
par=par,
identifiers=identifiers,
dir=directory,
filename=filename)
return_value = {}
return_value["files"] = list_to_string(files)
return return_value