def _update_girder(taskflow, body):
taskflow = to_taskflow(taskflow)
taskflow_id = taskflow['id']
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
client = _create_girder_client(girder_api_url, girder_token)
# If this is a retry then we have already create a task get it from
# the current tasks headers.
if body['retries'] > 0:
taskflow_task_id = \
current_task.request.headers[TASKFLOW_TASK_ID_HEADER]
# Celery always fires the postrun handler with a state of SUCCESS
# for retries. So we need to save the retries here so we can
# determine in the postrun handler if the task is really complete.
current_task.request.headers[TASKFLOW_RETRY_HEADER] \
= body['retries']
else:
# This is a new task so create a taskflow task instance
body = {
'celeryTaskId': body['id'],
'name': body['task']
}
url = 'taskflows/%s/tasks' % taskflow_id
r = client.post(url, data=json.dumps(body))
taskflow_task_id = r['_id']
return taskflow, taskflow_task_id
def _update_girder(taskflow, body):
taskflow = to_taskflow(taskflow)
taskflow_id = taskflow['id']
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
client = _create_girder_client(girder_api_url, girder_token)
# If this is a retry then we have already create a task get it from
# the current tasks headers.
if body['retries'] > 0:
taskflow_task_id = \
current_task.request.headers[TASKFLOW_TASK_ID_HEADER]
# Celery always fires the postrun handler with a state of SUCCESS
# for retries. So we need to save the retries here so we can
# determine in the postrun handler if the task is really complete.
current_task.request.headers[TASKFLOW_RETRY_HEADER] \
= body['retries']
else:
# This is a new task so create a taskflow task instance
body = {'celeryTaskId': body['id'], 'name': body['task']}
url = 'taskflows/%s/tasks' % taskflow_id
r = client.post(url, data=json.dumps(body))
taskflow_task_id = r['_id']
return taskflow, taskflow_task_id
def download_path(cluster_connection,
girder_token,
parent,
path,
assetstore_url,
assetstore_id,
upload=False,
include=None,
exclude=None):
"""
Download a given path on a cluster into an assetstore.
:params cluster_connection: The cluster connection to access the cluster.
:params girder_token: The Girder token to use to access Girder.
:params parent: The target folder to import the path into.
:params path: The path on the cluster to download.
:params assetstore_url: The url for the assetstore to use for the import.
:params assetstore_id: The id of the asseststore to import into.
:params upload: Indicate if the import should upload the file data or just
the metadata, the default is False.
:params include: List of include regexs
:params exclude: List of exclude regexs,
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
_import_path(cluster_connection,
girder_client,
parent,
path,
assetstore_url,
assetstore_id,
upload=upload,
include=include,
exclude=exclude)
def _taskflow_task_finished(taskflow, taskflow_task_id):
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)
return client.put(url)
def _taskflow_task_finished(taskflow, taskflow_task_id):
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)
return client.put(url)
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url',
required=True,
default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('-c',
default='remote',
choices=['remote', 'direct'],
help='command to run')
parser.add_argument('--foreground', dest='foreground', action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder', help='Girder\'s folder id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
else:
print('No implementation for command %s' % args.c)
def upload_file(cluster, girder_token, file, path):
"""
Upload a file to a cluster
:param cluster: The cluster to upload to.
:param girder_tokebn: The Grider token for Girder access.
:param file: The Girder file object.
:param path: The path on the cluster to upload to.
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
with get_connection(girder_token, cluster) as conn:
conn.makedirs(os.path.dirname(path))
_upload_file(conn, girder_client, file, path)
def upload_file(cluster, girder_token, file, path):
"""
Upload a file to a cluster
:param cluster: The cluster to upload to.
:param girder_tokebn: The Grider token for Girder access.
:param file: The Girder file object.
:param path: The path on the cluster to upload to.
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
with get_connection(girder_token, cluster) as conn:
conn.makedirs(os.path.dirname(path))
_upload_file(conn, girder_client, file, path)
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url', required=True, default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('-c', default='remote', choices=['remote', 'direct'],
help='command to run')
parser.add_argument('--foreground', dest='foreground',
action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder', help='Girder\'s folder id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder,
foreground=args.foreground, ro=True, allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder,
foreground=args.foreground, ro=True, allow_other=True)
else:
print('No implementation for command %s' % args.c)
def download_path(cluster_connection, girder_token, parent, path,
assetstore_url, assetstore_id, upload=False, include=None,
exclude=None):
"""
Download a given path on a cluster into an assetstore.
:params cluster_connection: The cluster connection to access the cluster.
:params girder_token: The Girder token to use to access Girder.
:params parent: The target folder to import the path into.
:params path: The path on the cluster to download.
:params assetstore_url: The url for the assetstore to use for the import.
:params assetstore_id: The id of the asseststore to import into.
:params upload: Indicate if the import should upload the file data or just
the metadata, the default is False.
:params include: List of include regexs
:params exclude: List of exclude regexs,
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
_import_path(cluster_connection, girder_client, parent, path,
assetstore_url, assetstore_id, upload=upload, include=include,
exclude=exclude)
def _create_girder_client(girder_api_url, girder_token):
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
return client
def client(self) -> GirderClient:
cl = GirderClient(apiUrl=self.api)
cl.token = self.token
return cl
def upload_path(cluster_connection, girder_token, folder_id, path):
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
cluster_connection.makedirs(path)
_upload_path(cluster_connection, girder_client, folder_id, path)
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url',
required=True,
default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('--foreground', dest='foreground', action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('-c',
default='remote',
help='command to run',
choices=['remote', 'direct', 'wt_dms', 'wt_home'])
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder',
help='Girder\'s folder id or a DM session id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'wt_dms':
FUSE(WtDmsGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'wt_home':
user = gc.get('/user/me')
args = {
'user': user['login'],
'pass': '******'.format(gc.token),
'dest': args.local_folder,
'opts': '-o uid=1000,gid=100', # FIXME
'url': gc.urlBase.replace('api/v1', 'homes').rstrip('/') # FIXME
}
cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}'
cmd = cmd.format(**args)
subprocess.check_output(cmd, shell=True) # FIXME
else:
print('No implementation for command %s' % args.c)
def __init__(self, session_id, api_url, token):
super().__init__()
self.session_id = session_id
gc = GirderClient(apiUrl=api_url)
gc.token = token
self._fs = WtDmsGirderFS(session_id, gc)
def main(args=None):
parser = argparse.ArgumentParser(description="Mount Girder filesystem assetstore.")
parser.add_argument(
"--api-url",
required=True,
default=None,
help="full URL to the RESTful API of Girder server",
)
parser.add_argument("--username", required=False, default=None)
parser.add_argument("--password", required=False, default=None)
parser.add_argument("--api-key", required=False, default=None)
parser.add_argument("--token", required=False, default=None)
parser.add_argument("--foreground", dest="foreground", action="store_true")
parser.add_argument("--hostns", dest="hostns", action="store_true")
parser.add_argument(
"--versions-mountpoint",
dest="versions_mountpoint",
required=False,
help="Mountpoint for the versions FS. If relative, then it should be "
"relative to the runs mountpoint",
default="Versions",
)
parser.add_argument(
"-c",
default="remote",
help="type of filesystem to mount",
choices=[
"remote",
"direct",
"wt_dms",
"wt_home",
"wt_work",
"wt_run",
"wt_versions",
"wt_runs",
],
)
parser.add_argument("local_folder", help="path to local target folder")
parser.add_argument(
"remote_folder",
help="Girder's folder id, a DM session id (for wt_dms), or a tale instance"
"ID (for wt_versions)",
)
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get("HOSTDIR", "/"), "proc/1/ns/mnt")
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == "remote":
FUSE(
RESTGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True,
)
elif args.c == "direct":
FUSE(
LocalGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True,
)
elif args.c == "wt_dms":
FUSE(
WtDmsGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True,
)
elif args.c == "wt_run":
user = gc.get("/user/me")
args = {
"user": user["login"],
"pass": "******".format(gc.token),
"dest": args.local_folder,
"runId": args.remote_folder,
"opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME
"url": gc.urlBase.replace("api/v1", "runs").rstrip("/"), # FIXME
}
cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{runId} {dest}'
cmd = cmd.format(**args)
subprocess.check_output(cmd, shell=True) # FIXME
elif args.c == "wt_work":
user = gc.get("/user/me")
args = {
"user": user["login"],
"pass": "******".format(gc.token),
"dest": args.local_folder,
"tale": args.remote_folder,
"opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME
"url": gc.urlBase.replace("api/v1", "tales").rstrip("/"), # FIXME
}
cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{tale} {dest}'
cmd = cmd.format(**args)
subprocess.check_output(cmd, shell=True) # FIXME
elif args.c == "wt_home":
user = gc.get("/user/me")
args = {
"user": user["login"],
"pass": "******".format(gc.token),
"dest": args.local_folder,
"opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME
"url": gc.urlBase.replace("api/v1", "homes").rstrip("/"), # FIXME
}
cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}'
cmd = cmd.format(**args)
subprocess.check_output(cmd, shell=True) # FIXME
elif args.c == "wt_versions":
FUSE(
WtVersionsFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=False,
allow_other=True,
)
elif args.c == "wt_runs":
FUSE(
WtRunsFS(args.remote_folder, gc, args.versions_mountpoint),
args.local_folder,
foreground=args.foreground,
ro=False,
allow_other=True,
)
else:
print("No implementation for command %s" % args.c)
def upload_path(cluster_connection, girder_token, folder_id, path):
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
cluster_connection.makedirs(path)
_upload_path(cluster_connection, girder_client, folder_id, path)
girder_api_key = os.environ.get('GIRDER_API_KEY')
girder_token = os.environ.get('GIRDER_TOKEN')
app_base_url = os.environ.get('APP_BASE_URL')
cluster_id = os.environ.get('CLUSTER_ID')
jupyterhub_base_url = os.environ.get('JUPYTERHUB_BASE_URL')
if girder_host:
girder_client = GirderClient(host=girder_host,
port=girder_port,
scheme=girder_scheme,
apiRoot=girder_api_root)
if girder_api_key is not None:
girder_client.authenticate(apiKey=girder_api_key)
elif girder_token is not None:
girder_client.token = girder_token
girder_file = lookup_file(girder_client, jupyterhub_base_url)
# TODO Need to use basis and theory
def _fetch_calculation(molecule_id,
type_=None,
basis=None,
theory=None,
functional=None):
parameters = {'moleculeId': molecule_id, 'sortByTheory': True}
if type_ is not None:
parameters['calculationType'] = type_
def createGirderClient(requestInfo):
"""Return new configured GirderClient instance."""
gc = GirderClient(apiUrl=requestInfo.apiUrl)
gc.token = requestInfo.token['_id']
return gc
def _create_girder_client(girder_api_url, girder_token):
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
return client
def create_task_job(job_defaults,
sender=None,
body=None,
exchange=None,
routing_key=None,
headers=None,
properties=None,
declare=None,
retry_policy=None,
**kwargs):
parent_task = current_app.current_task
try:
if parent_task is None:
raise MissingJobArguments('Parent task is None')
if parent_task.request is None:
raise MissingJobArguments("Parent task's request is None")
if not hasattr(parent_task.request, 'girder_api_url'):
raise MissingJobArguments(
"Parent task's request does not contain girder_api_url")
if not hasattr(parent_task.request, 'girder_client_token'):
raise MissingJobArguments(
"Parent task's request does not contain girder_client_token")
if not hasattr(parent_task.request, 'id'):
raise MissingJobArguments(
"Parent task's request does not contain id")
if 'id' not in headers:
raise MissingJobArguments('id is not in headers')
gc = GirderClient(apiUrl=parent_task.request.girder_api_url)
gc.token = parent_task.request.girder_client_token
task_args = tuple(_walk_obj(body[0], _maybe_model_repr))
task_kwargs = _walk_obj(body[1], _maybe_model_repr)
parameters = {
'title':
headers.pop('girder_job_title',
job_defaults.get('girder_job_title', '')),
'type':
headers.pop('girder_job_type',
job_defaults.get('girder_job_type', '')),
'handler':
headers.pop('girder_job_handler',
job_defaults.get('girder_job_handler', '')),
'public':
headers.pop('girder_job_public',
job_defaults.get('girder_job_public', '')),
'args':
json.dumps(task_args),
'kwargs':
task_kwargs,
'otherFields':
json.dumps(
dict(celeryTaskId=headers['id'],
celeryParentTaskId=parent_task.request.id,
**headers.pop(
'girder_job_other_fields',
job_defaults.get('girder_job_other_fields', ''))))
}
try:
response = gc.post('job', parameters=parameters, jsonResp=False)
if response.ok:
headers['jobInfoSpec'] = response.json().get('jobInfoSpec')
except requests.exceptions.RequestException as e:
logger.warn('Failed to post job: {}'.format(e))
except MissingJobArguments as e:
logger.warn('Girder job not created: {}'.format(str(e)))
lon_name = d
contour_data = {
'gridWidth': lon_select_index,
'gridHeight': lat_select_index,
'x0': float(data.variables[lon_name][0]),
'y0': float(data.variables[lat_name][0]),
'dx': float(data.variables[lon_name][1] - data.variables[lon_name][0]),
'dy': float(data.variables[lat_name][1] - data.variables[lat_name][0]),
'values': variable[timestep][:lat_select_index, :lon_select_index].reshape(variable[timestep][:lat_select_index, :lon_select_index].size).tolist()
}
return contour_data
client = GirderClient(host, port)
client.token = token
# Get the user
user = client.get('user/me')
# Get the dataset folder
parameters = {
'userId': user['_id']
}
dataset_folder = client.get('minerva_dataset/folder', parameters=parameters)['folder']
dataset_folder_id = dataset_folder['_id']
parameters = {
'id': fileId,
'type': 'file'
}
# Get the file resource so we can get the name
