def _update_girder(taskflow, body): taskflow = to_taskflow(taskflow) taskflow_id = taskflow['id'] girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token client = _create_girder_client(girder_api_url, girder_token) # If this is a retry then we have already create a task get it from # the current tasks headers. if body['retries'] > 0: taskflow_task_id = \ current_task.request.headers[TASKFLOW_TASK_ID_HEADER] # Celery always fires the postrun handler with a state of SUCCESS # for retries. So we need to save the retries here so we can # determine in the postrun handler if the task is really complete. current_task.request.headers[TASKFLOW_RETRY_HEADER] \ = body['retries'] else: # This is a new task so create a taskflow task instance body = { 'celeryTaskId': body['id'], 'name': body['task'] } url = 'taskflows/%s/tasks' % taskflow_id r = client.post(url, data=json.dumps(body)) taskflow_task_id = r['_id'] return taskflow, taskflow_task_id
def _update_girder(taskflow, body): taskflow = to_taskflow(taskflow) taskflow_id = taskflow['id'] girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token client = _create_girder_client(girder_api_url, girder_token) # If this is a retry then we have already create a task get it from # the current tasks headers. if body['retries'] > 0: taskflow_task_id = \ current_task.request.headers[TASKFLOW_TASK_ID_HEADER] # Celery always fires the postrun handler with a state of SUCCESS # for retries. So we need to save the retries here so we can # determine in the postrun handler if the task is really complete. current_task.request.headers[TASKFLOW_RETRY_HEADER] \ = body['retries'] else: # This is a new task so create a taskflow task instance body = {'celeryTaskId': body['id'], 'name': body['task']} url = 'taskflows/%s/tasks' % taskflow_id r = client.post(url, data=json.dumps(body)) taskflow_task_id = r['_id'] return taskflow, taskflow_task_id
def download_path(cluster_connection, girder_token, parent, path, assetstore_url, assetstore_id, upload=False, include=None, exclude=None): """ Download a given path on a cluster into an assetstore. :params cluster_connection: The cluster connection to access the cluster. :params girder_token: The Girder token to use to access Girder. :params parent: The target folder to import the path into. :params path: The path on the cluster to download. :params assetstore_url: The url for the assetstore to use for the import. :params assetstore_id: The id of the asseststore to import into. :params upload: Indicate if the import should upload the file data or just the metadata, the default is False. :params include: List of include regexs :params exclude: List of exclude regexs, """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token _import_path(cluster_connection, girder_client, parent, path, assetstore_url, assetstore_id, upload=upload, include=include, exclude=exclude)
def _taskflow_task_finished(taskflow, taskflow_task_id): girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id) return client.put(url)
def _taskflow_task_finished(taskflow, taskflow_task_id): girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id) return client.put(url)
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('-c', default='remote', choices=['remote', 'direct'], help='command to run') parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) else: print('No implementation for command %s' % args.c)
def upload_file(cluster, girder_token, file, path): """ Upload a file to a cluster :param cluster: The cluster to upload to. :param girder_tokebn: The Grider token for Girder access. :param file: The Girder file object. :param path: The path on the cluster to upload to. """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token with get_connection(girder_token, cluster) as conn: conn.makedirs(os.path.dirname(path)) _upload_file(conn, girder_client, file, path)
def upload_file(cluster, girder_token, file, path): """ Upload a file to a cluster :param cluster: The cluster to upload to. :param girder_tokebn: The Grider token for Girder access. :param file: The Girder file object. :param path: The path on the cluster to upload to. """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token with get_connection(girder_token, cluster) as conn: conn.makedirs(os.path.dirname(path)) _upload_file(conn, girder_client, file, path)
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('-c', default='remote', choices=['remote', 'direct'], help='command to run') parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) else: print('No implementation for command %s' % args.c)
def download_path(cluster_connection, girder_token, parent, path, assetstore_url, assetstore_id, upload=False, include=None, exclude=None): """ Download a given path on a cluster into an assetstore. :params cluster_connection: The cluster connection to access the cluster. :params girder_token: The Girder token to use to access Girder. :params parent: The target folder to import the path into. :params path: The path on the cluster to download. :params assetstore_url: The url for the assetstore to use for the import. :params assetstore_id: The id of the asseststore to import into. :params upload: Indicate if the import should upload the file data or just the metadata, the default is False. :params include: List of include regexs :params exclude: List of exclude regexs, """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token _import_path(cluster_connection, girder_client, parent, path, assetstore_url, assetstore_id, upload=upload, include=include, exclude=exclude)
def _create_girder_client(girder_api_url, girder_token): client = GirderClient(apiUrl=girder_api_url) client.token = girder_token return client
def client(self) -> GirderClient: cl = GirderClient(apiUrl=self.api) cl.token = self.token return cl
def upload_path(cluster_connection, girder_token, folder_id, path): girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token cluster_connection.makedirs(path) _upload_path(cluster_connection, girder_client, folder_id, path)
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('-c', default='remote', help='command to run', choices=['remote', 'direct', 'wt_dms', 'wt_home']) parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id or a DM session id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'wt_dms': FUSE(WtDmsGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'wt_home': user = gc.get('/user/me') args = { 'user': user['login'], 'pass': '******'.format(gc.token), 'dest': args.local_folder, 'opts': '-o uid=1000,gid=100', # FIXME 'url': gc.urlBase.replace('api/v1', 'homes').rstrip('/') # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME else: print('No implementation for command %s' % args.c)
def __init__(self, session_id, api_url, token): super().__init__() self.session_id = session_id gc = GirderClient(apiUrl=api_url) gc.token = token self._fs = WtDmsGirderFS(session_id, gc)
def main(args=None): parser = argparse.ArgumentParser(description="Mount Girder filesystem assetstore.") parser.add_argument( "--api-url", required=True, default=None, help="full URL to the RESTful API of Girder server", ) parser.add_argument("--username", required=False, default=None) parser.add_argument("--password", required=False, default=None) parser.add_argument("--api-key", required=False, default=None) parser.add_argument("--token", required=False, default=None) parser.add_argument("--foreground", dest="foreground", action="store_true") parser.add_argument("--hostns", dest="hostns", action="store_true") parser.add_argument( "--versions-mountpoint", dest="versions_mountpoint", required=False, help="Mountpoint for the versions FS. If relative, then it should be " "relative to the runs mountpoint", default="Versions", ) parser.add_argument( "-c", default="remote", help="type of filesystem to mount", choices=[ "remote", "direct", "wt_dms", "wt_home", "wt_work", "wt_run", "wt_versions", "wt_runs", ], ) parser.add_argument("local_folder", help="path to local target folder") parser.add_argument( "remote_folder", help="Girder's folder id, a DM session id (for wt_dms), or a tale instance" "ID (for wt_versions)", ) args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get("HOSTDIR", "/"), "proc/1/ns/mnt") with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == "remote": FUSE( RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "direct": FUSE( LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "wt_dms": FUSE( WtDmsGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "wt_run": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "runId": args.remote_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "runs").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{runId} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_work": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "tale": args.remote_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "tales").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{tale} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_home": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "homes").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_versions": FUSE( WtVersionsFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=False, allow_other=True, ) elif args.c == "wt_runs": FUSE( WtRunsFS(args.remote_folder, gc, args.versions_mountpoint), args.local_folder, foreground=args.foreground, ro=False, allow_other=True, ) else: print("No implementation for command %s" % args.c)
def upload_path(cluster_connection, girder_token, folder_id, path): girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token cluster_connection.makedirs(path) _upload_path(cluster_connection, girder_client, folder_id, path)
girder_api_key = os.environ.get('GIRDER_API_KEY') girder_token = os.environ.get('GIRDER_TOKEN') app_base_url = os.environ.get('APP_BASE_URL') cluster_id = os.environ.get('CLUSTER_ID') jupyterhub_base_url = os.environ.get('JUPYTERHUB_BASE_URL') if girder_host: girder_client = GirderClient(host=girder_host, port=girder_port, scheme=girder_scheme, apiRoot=girder_api_root) if girder_api_key is not None: girder_client.authenticate(apiKey=girder_api_key) elif girder_token is not None: girder_client.token = girder_token girder_file = lookup_file(girder_client, jupyterhub_base_url) # TODO Need to use basis and theory def _fetch_calculation(molecule_id, type_=None, basis=None, theory=None, functional=None): parameters = {'moleculeId': molecule_id, 'sortByTheory': True} if type_ is not None: parameters['calculationType'] = type_
def createGirderClient(requestInfo): """Return new configured GirderClient instance.""" gc = GirderClient(apiUrl=requestInfo.apiUrl) gc.token = requestInfo.token['_id'] return gc
def _create_girder_client(girder_api_url, girder_token): client = GirderClient(apiUrl=girder_api_url) client.token = girder_token return client
def create_task_job(job_defaults, sender=None, body=None, exchange=None, routing_key=None, headers=None, properties=None, declare=None, retry_policy=None, **kwargs): parent_task = current_app.current_task try: if parent_task is None: raise MissingJobArguments('Parent task is None') if parent_task.request is None: raise MissingJobArguments("Parent task's request is None") if not hasattr(parent_task.request, 'girder_api_url'): raise MissingJobArguments( "Parent task's request does not contain girder_api_url") if not hasattr(parent_task.request, 'girder_client_token'): raise MissingJobArguments( "Parent task's request does not contain girder_client_token") if not hasattr(parent_task.request, 'id'): raise MissingJobArguments( "Parent task's request does not contain id") if 'id' not in headers: raise MissingJobArguments('id is not in headers') gc = GirderClient(apiUrl=parent_task.request.girder_api_url) gc.token = parent_task.request.girder_client_token task_args = tuple(_walk_obj(body[0], _maybe_model_repr)) task_kwargs = _walk_obj(body[1], _maybe_model_repr) parameters = { 'title': headers.pop('girder_job_title', job_defaults.get('girder_job_title', '')), 'type': headers.pop('girder_job_type', job_defaults.get('girder_job_type', '')), 'handler': headers.pop('girder_job_handler', job_defaults.get('girder_job_handler', '')), 'public': headers.pop('girder_job_public', job_defaults.get('girder_job_public', '')), 'args': json.dumps(task_args), 'kwargs': task_kwargs, 'otherFields': json.dumps( dict(celeryTaskId=headers['id'], celeryParentTaskId=parent_task.request.id, **headers.pop( 'girder_job_other_fields', job_defaults.get('girder_job_other_fields', '')))) } try: response = gc.post('job', parameters=parameters, jsonResp=False) if response.ok: headers['jobInfoSpec'] = response.json().get('jobInfoSpec') except requests.exceptions.RequestException as e: logger.warn('Failed to post job: {}'.format(e)) except MissingJobArguments as e: logger.warn('Girder job not created: {}'.format(str(e)))
lon_name = d contour_data = { 'gridWidth': lon_select_index, 'gridHeight': lat_select_index, 'x0': float(data.variables[lon_name][0]), 'y0': float(data.variables[lat_name][0]), 'dx': float(data.variables[lon_name][1] - data.variables[lon_name][0]), 'dy': float(data.variables[lat_name][1] - data.variables[lat_name][0]), 'values': variable[timestep][:lat_select_index, :lon_select_index].reshape(variable[timestep][:lat_select_index, :lon_select_index].size).tolist() } return contour_data client = GirderClient(host, port) client.token = token # Get the user user = client.get('user/me') # Get the dataset folder parameters = { 'userId': user['_id'] } dataset_folder = client.get('minerva_dataset/folder', parameters=parameters)['folder'] dataset_folder_id = dataset_folder['_id'] parameters = { 'id': fileId, 'type': 'file' } # Get the file resource so we can get the name