def download_path(cluster_connection, girder_token, parent, path, assetstore_url, assetstore_id, upload=False, include=None, exclude=None): """ Download a given path on a cluster into an assetstore. :params cluster_connection: The cluster connection to access the cluster. :params girder_token: The Girder token to use to access Girder. :params parent: The target folder to import the path into. :params path: The path on the cluster to download. :params assetstore_url: The url for the assetstore to use for the import. :params assetstore_id: The id of the asseststore to import into. :params upload: Indicate if the import should upload the file data or just the metadata, the default is False. :params include: List of include regexs :params exclude: List of exclude regexs, """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token _import_path(cluster_connection, girder_client, parent, path, assetstore_url, assetstore_id, upload=upload, include=include, exclude=exclude)
def __init__(self, girder_url, newt_sessionid): ''' ''' self._client = None self._cluster_id = None self._girder_url = girder_url self._input_folder_id = None self._job_folder_id = None self._job_id = None self._output_folder_id = None self._private_folder_id = None self._script_id = None self._session = requests.Session() # Authenticate with Girder using the newt session id url = '%s/api/v1/newt/authenticate/%s' % \ (self._girder_url, newt_sessionid) r = self._session.put(url) if r.status_code != 200: raise HttpError(r.status_code, r.text, r.url, r.request.method) # Instantiate Girder client url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') #print 'user', user user_id = user['_id'] r = self._client.listFolder(user_id, 'user', name='Private') if len(r) != 1: raise Exception('Wrong number of users; should be 1 got %s' % len(r)) self._private_folder_id = r[0]['_id'] print 'private_folder_id', self._private_folder_id
def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post('https://newt.nersc.gov/newt/auth', { 'username': self._girder_user, 'password': self._girder_password }) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()['newt_sessionid'] # Now authenticate with Girder using the session id url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') self._user_id = user['_id'] r = self._client.listFolder(self._user_id, 'user', name='Private') r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id']
def main(): parser = argparse.ArgumentParser( description='Import analyses into minerva') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--scheme', required=False, default='http') parser.add_argument('--host', required=False, default='localhost') parser.add_argument('--port', required=False, default='8080') parser.add_argument('--api-root', required=False, default='/api/v1', help='path to the Girder REST API') parser.add_argument('--path', required=True, help='the path to import the analyses from') config = parser.parse_args() client = GirderClient(host=config.host, port=config.port, apiRoot=config.api_root, scheme=config.scheme) client.authenticate(config.username, config.password) import_analyses(client, config.path)
def main(config): client = GirderClient(apiUrl=config.girder_api_url) client.authenticate(config.girder_user, config.girder_password) # Load any parameters params = {} if config.taskflow_start_params is not None: with open(config.taskflow_start_params) as fp: params = json.load(fp) print params try: print ('Running %s taskflow ...' % config.taskflow_start_params) taskflow_id = create_taskflow( client, config.taskflow_class) # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url, data=json.dumps(params)) # Wait for it to complete wait_for_complete(client, taskflow_id) except HttpError as ex: print( ex.responseText)
def _update_girder(taskflow, body): taskflow = to_taskflow(taskflow) taskflow_id = taskflow['id'] girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token client = _create_girder_client(girder_api_url, girder_token) # If this is a retry then we have already create a task get it from # the current tasks headers. if body['retries'] > 0: taskflow_task_id = \ current_task.request.headers[TASKFLOW_TASK_ID_HEADER] # Celery always fires the postrun handler with a state of SUCCESS # for retries. So we need to save the retries here so we can # determine in the postrun handler if the task is really complete. current_task.request.headers[TASKFLOW_RETRY_HEADER] \ = body['retries'] else: # This is a new task so create a taskflow task instance body = {'celeryTaskId': body['id'], 'name': body['task']} url = 'taskflows/%s/tasks' % taskflow_id r = client.post(url, data=json.dumps(body)) taskflow_task_id = r['_id'] return taskflow, taskflow_task_id
def test_sanity_checks(): gc = GirderClient(apiUrl=source_api_root) resp = gc.get('/', jsonResp=False) assert resp.status_code == 200 assert resp.headers['Content-Type'] == 'text/html;charset=utf-8' resp = gc.get('system/check', jsonResp=False) assert resp.status_code == 200
def get_girder_client() -> GirderClient: client = GirderClient(apiUrl="https://viame.kitware.com/api/v1") apikey = os.environ.get('GIRDER_API_KEY', None) if apikey: client.authenticate(apiKey=apikey) else: client.authenticate(interactive=True) return client
def import_calc(config): try: target_port = None if config.port: target_port = config.port target_scheme = None if config.scheme: target_scheme = config.scheme target_apiroot = None if config.apiroot: target_apiroot = config.apiroot client = GirderClient(host=config.host, port=target_port, scheme=target_scheme, apiRoot=target_apiroot) client.authenticate(apiKey=config.apiKey) me = client.get('/user/me') if not me: print('Error: Girder token invalid, please verify') return folderParams = { 'parentId': me['_id'], 'parentType': 'user', 'name': 'Private' } # Get the private folder id first folder = next(client.listResource('folder', folderParams)) folder = next(client.listFolder(me['_id'], 'user', 'Private')) for file_name in config.datafile: print ('\nUploading ' + file_name) file_id = {} with open(file_name, 'r') as fp: fileNameBase = os.path.basename(file_name) size = os.path.getsize(file_name) file_id = client.uploadFile(folder['_id'], fp, fileNameBase, size, 'folder') body = { 'fileId': file_id['_id'] } if config.public: body['public'] = True mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body)) if mol and '_id' in mol: config.moleculeId = mol['_id'] print('Molecule ID: ' + mol['_id']) else: print(mol) except HttpError as error: print(error.responseText, file=sys.stderr)
def _taskflow_task_finished(taskflow, taskflow_task_id): girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id) return client.put(url)
def test_extract_download(data): fileId, filename = data filepath = localDataRoot / str(filename) if not filepath.exists(): gc = GirderClient(apiUrl=source_api_root) gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY')) gc.downloadFile(fileId, str(filepath)) with zipfile.ZipFile(filepath, 'r') as zipref: zipref.extractall(localDataRoot)
def main(ctx, api_key, api_url): """Openchemistry Client The client can be used to fetch molecules, add molecules, etc. """ gc = GirderClient(apiUrl=api_url) if api_key is not None: gc.authenticate(apiKey=api_key) ctx.obj = gc
def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id']
def gc_init(api_url='https://girder.hub.yt/api/v1', api_key=None): from girder_client import GirderClient if api_key is None: try: api_key = os.environ['GIRDER_API_KEY'] except KeyError as err: msg = 'please define GIRDER_API_KEY environment variable\n' msg += ' or pass api_key kwargs.' raise KeyError(msg) gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) return gc
def _ingest(project, composite, dir, channel_map, api_url, api_key): if dir[-1] != '/': dir += '/' gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) channel_map = json.load(channel_map) channel_map = {channel.upper():element.lower() for (channel,element) in channel_map.items()} experiments = _ingest_runs(gc, project, composite, dir) #(run_ids, runs) = _ingest_runs(gc, project, composite, dir) _ingest_samples(gc, project, composite, dir, experiments, channel_map)
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('-c', default='remote', choices=['remote', 'direct'], help='command to run') parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) else: print('No implementation for command %s' % args.c)
def upload_file(cluster, girder_token, file, path): """ Upload a file to a cluster :param cluster: The cluster to upload to. :param girder_tokebn: The Grider token for Girder access. :param file: The Girder file object. :param path: The path on the cluster to upload to. """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token with get_connection(girder_token, cluster) as conn: conn.makedirs(os.path.dirname(path)) _upload_file(conn, girder_client, file, path)
def testClientMetadataExtractor(testData, user): item = Item().load(testData['item']['_id'], user=user) assert item['name'] == testData['name'] del item['meta'] item = Item().save(item) assert 'meta' not in item client = GirderClient('localhost', int(os.environ['GIRDER_PORT'])) client.authenticate(user['login'], 'password') extractor = ClientMetadataExtractor(client, testData['path'], testData['item']['_id']) extractor.extractMetadata() item = Item().load(testData['item']['_id'], user=user) assert item['name'] == testData['name'] assert 'meta' in item assert item['meta']['MIME type'] == testData['mimeType']
def init_girder(api_key=None, api_url='https://girder.hub.yt/api/v1'): """Initialize girder client, rely on environment variable: GIRDER_API_KEY, connect to yt Hub. Args: api_key (str, optional): use GIRDER_API_KEY env. var. by default api_url (str, optional): use yt Hub v1 by default Return: GirderClient: initialized girder client """ from girder_client import GirderClient if api_key is None: import os api_key = os.environ['GIRDER_API_KEY'] gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) return gc
def upload_benchmark_results(benchmark_bin, api_key=None): hostname = socket.gethostname().lower() results_dir = os.path.join(benchmark_bin, 'BenchmarkResults', hostname) if not os.path.exists(results_dir): sys.stderr.write('Expected results directory does not exist: ' + results_dir) sys.exit(1) from girder_client import GirderClient gc = GirderClient(apiUrl='https://data.kitware.com/api/v1') gc.authenticate(apiKey=api_key) # ITK/PerformanceBenchmarkingResults folder_id = '5af50c818d777f06857985e3' hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder') gc.upload(os.path.join(results_dir, '*.json'), hostname_folder['_id'], leafFoldersAsItems=False, reuseExisting=True)
def create_movie(id): token = request.headers.get('girderToken', '') if not token or not id: return Response('Invalid token or parameter ID.', status=400) gc = GirderClient(apiUrl=girder_url) gc.setToken(token) with tempfile.TemporaryDirectory() as tmpdir: gc.downloadItem(id, tmpdir) item_name = os.listdir(tmpdir)[0] path_name = os.path.join(tmpdir, item_name, '*.svg') output_file = tempfile.NamedTemporaryFile(suffix='.mp4') (ffmpeg .input(path_name, pattern_type='glob', framerate=10) .output(output_file.name) .overwrite_output() .run()) return send_file(output_file, mimetype='mp4')
def testClientMetadataExtractor(self): item = Item().load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertNotHasKeys(item, ['meta']) clientPath = os.path.join(ROOT_DIR, 'clients', 'python') sys.path.insert(0, clientPath) from girder_client import GirderClient client = GirderClient('localhost', int(os.environ['GIRDER_PORT'])) client.authenticate(self.user['login'], self.password) extractor = ClientMetadataExtractor(client, self.path, self.item['_id']) extractor.extractMetadata() sys.path.remove(clientPath) item = Item().load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertHasKeys(item, ['meta']) self.assertEqual(item['meta']['MIME type'], self.mimeType)
def gw_task_prerun(task=None, sender=None, task_id=None, args=None, kwargs=None, **rest): """Deserialize the jobInfoSpec passed in through the headers. This provides the a JobManager class as an attribute of the task before task execution. decorated functions may bind to their task and have access to the job_manager for logging and updating their status in girder. """ if is_builtin_celery_task(sender.name): return try: task.job_manager = _job_manager(task.request, task.request.headers) _update_status(task, JobStatus.RUNNING) except JobSpecNotFound: task.job_manager = None logger.warn('No jobInfoSpec. Setting job_manager to None.') except StateTransitionException: # Fetch the current status of the job status = task.job_manager.refreshStatus() # If we are canceling we want to stay in that state if status != JobStatus.CANCELING: raise try: task.girder_client = GirderClient(apiUrl=task.request.girder_api_url) task.girder_client.token = task.request.girder_client_token except AttributeError: task.girder_client = None # Deserialize girder_result_hooks if they exist if hasattr(task.request, 'girder_result_hooks'): u = jsonpickle.unpickler.Unpickler() task.request.girder_result_hooks = \ [u.restore(grh) for grh in task.request.girder_result_hooks]
def test_upload_zip_data(dataset: dict): user = zipUser client = GirderClient(apiUrl='http://localhost:8010/api/v1') client.authenticate(username=user['login'], password=user['password']) dsPath = localDataRoot / str(dataset['path']) privateFolder = getTestFolder(client) newDatasetFolder = client.createFolder( privateFolder['_id'], dataset['name'], metadata={ 'fps': dataset['fps'], 'type': dataset['type'], }, ) if Path(dsPath).is_file(): client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath)) client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}') wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status']) resultFolder = client.getFolder(newDatasetFolder['_id']) # verify sub datasets if they exist if dataset.get('subDatasets', False): folders = list(client.listFolder(newDatasetFolder['_id'])) for item in dataset["subDatasets"]: matches = [x for x in folders if x["name"] == item["name"]] if len(matches) > 0: meta = matches[0].get("meta", {}) assert meta.get("fps", -1) == item["fps"] assert meta.get("type", "") == item["type"] assert meta.get("annotate", False) elif dataset['job_status'] == JobStatus.SUCCESS: assert resultFolder['meta'].get("annotate", False) assert type(resultFolder['meta'].get("fps")) in [int, float] assert type(resultFolder['meta'].get("type")) == str else: assert resultFolder['meta'].get("annotate", None) is None
def __init__(self, *args, **kwargs): gc = kwargs.pop('gc', None) try: if gc is None: # We need to resolve Girder's API URL, but girder_worker can # specify a different value than what Girder gets from a rest # request. # Girder 3 try: from girder_worker.girder_plugin.utils import getWorkerApiUrl except ImportError: # Girder 2 try: from girder.plugins.worker.utils import getWorkerApiUrl # Fall back if the worker plugin is unavailble except ImportError: from girder.api.rest import getApiUrl as getWorkerApiUrl self.gc = GirderClient(apiUrl=getWorkerApiUrl()) from girder.api.rest import getCurrentUser if getCurrentUser(): from girder.constants import TokenScope from girder.models.token import Token token = Token().createToken( days=7, scope=[TokenScope.DATA_READ, TokenScope.DATA_WRITE], user=getCurrentUser(), )['_id'] else: from girder.api.rest import getCurrentToken token = getCurrentToken()['_id'] self.gc.token = token else: self.gc = gc except ImportError: self.gc = None
from girder_client import GirderClient import json import pymongo import sys if len(sys.argv) < 2: print "%s /path/to/ArborWebApps" % sys.argv[0] sys.exit(1) arborWebAppsPath = sys.argv[1] # Get the ID for our Analyses folder. c = GirderClient(host='localhost', port=9000) c.authenticate('girder', 'girder') folderSearch = c.get('resource/search', parameters={ 'q': 'Analyses', 'types': '["folder"]' }) folderId = folderSearch['folder'][0]['_id'] # Disable authorization requirements for running romanesco tasks c.put('system/setting', parameters={ 'key': 'romanesco.require_auth', 'value': 'false' }) # Check if these analyses already exist. If so, we won't re-upload them. uploadACR = False uploadPGS = False
parser.add_argument("path", type=str, help="path to Arbor web apps") parser.add_argument("-g", "--girder-host", type=str, default='localhost', help="host to Girder instance") parser.add_argument("-p", "--girder-port", type=int, default=9000, help="port to Girder instance") args = parser.parse_args() # Get the ID for our Analyses folder. c = GirderClient(host=args.girder_host, port=args.girder_port) c.authenticate('girder', 'girder') folderSearch = c.get('resource/search', parameters={ 'q': 'Analyses', 'types': '["folder"]' }) folderId = folderSearch['folder'][0]['_id'] # Disable authorization requirements for running romanesco tasks c.put('system/setting', parameters={ 'key': 'flow.require_auth', 'value': 'false' })
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### from girder_client import GirderClient if __name__ == "__main__": login = '******' password = '******' gc = GirderClient(apiUrl='http://*****:*****@admin.com', firstName='admin', lastName='admin', password=password, admin=True) gc.authenticate(username=login, password=password) # Create an assetstore gc.post('assetstore', parameters={ 'name': 'TestAssetstore', 'type': 0,
def girder_client(request, api_url): username, password = request.param client = GirderClient(apiUrl=api_url) client.authenticate(username, password) yield client
return result parser = ArgumentParser(description='Initialize the girder environment') parser.add_argument('--admin', help='name:pass for the admin user') parser.add_argument('--host', help='host to connect to') parser.add_argument('--port', type=int, help='port to connect to') parser.add_argument('--broker', help='girder worker broker URI') parser.add_argument('--s3', help='name of S3 bucket') parser.add_argument('--aws-key-id', help='aws key id') parser.add_argument('--aws-secret-key', help='aws secret key') args = parser.parse_args() client = GirderClient(host=args.host, port=args.port) user, password = args.admin.split(":", 1) if find_user('girder'): client.authenticate('girder', 'girder') ensure_user(client, login=user, password=password, email='*****@*****.**', firstName='Girder', lastName='Admin') client.authenticate(user, password)