class NewtIntegrationTest(BaseIntegrationTest): def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60 * 5): super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout) self._cluster_id = None self._machine = machine def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post( "https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password} ) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()["newt_sessionid"] # Now authenticate with Girder using the session id url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1" % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies["girderToken"] user = self._client.get("user/me") self._user_id = user["_id"] r = self._client.listFolder(self._user_id, "user", name="Private") r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]["_id"] def tearDown(self): super(NewtIntegrationTest, self).tearDown() if self._cluster_id: try: url = "clusters/%s" % self._cluster_id self._client.delete(url) except Exception: traceback.print_exc() def create_cluster(self): body = {"config": {"host": self._machine}, "name": "NewtIntegrationTest", "type": "newt"} r = self._client.post("clusters", data=json.dumps(body)) self._cluster_id = r["_id"] # Now test the connection r = self._client.put("clusters/%s/start" % self._cluster_id) sleeps = 0 while True: time.sleep(1) r = self._client.get("clusters/%s/status" % self._cluster_id) if r["status"] == "running": break elif r["status"] == "error": r = self._client.get("clusters/%s/log" % self._cluster_id) self.fail(str(r)) if sleeps > 9: self.fail("Cluster never moved into running state") sleeps += 1 def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i["name"].startswith("CumulusIntegrationTestJob-%s.o" % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get("item/%s/files" % i["_id"]) self.assertEqual(len(r), 1) url = "%s/api/v1/file/%s/download" % (self._girder_url, r[0]["_id"]) r = self._session.get(url) self.assertEqual(r.content, self._data) def test(self): try: self.create_cluster() self.create_script() self.create_input() self.create_output_folder() self.create_job() self.submit_job(timeout=self._job_timeout) self.assert_output() except HttpError as error: self.fail(error.responseText)
def __init__(self, apiKey, objectStore): """initialization function to create a GirderCli instance, will attempt to authenticate with the designated Girder instance. """ GirderClient.__init__(self, apiUrl="https://data.kitware.com/api/v1") self.objectStore = objectStore self.authenticate(apiKey=apiKey)
def __init__(self, girder_url, newt_sessionid): ''' ''' self._client = None self._cluster_id = None self._girder_url = girder_url self._input_folder_id = None self._job_folder_id = None self._job_id = None self._output_folder_id = None self._private_folder_id = None self._script_id = None self._session = requests.Session() # Authenticate with Girder using the newt session id url = '%s/api/v1/newt/authenticate/%s' % \ (self._girder_url, newt_sessionid) r = self._session.put(url) if r.status_code != 200: raise HttpError(r.status_code, r.text, r.url, r.request.method) # Instantiate Girder client url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') #print 'user', user user_id = user['_id'] r = self._client.listFolder(user_id, 'user', name='Private') if len(r) != 1: raise Exception('Wrong number of users; should be 1 got %s' % len(r)) self._private_folder_id = r[0]['_id'] print 'private_folder_id', self._private_folder_id
def download_path(cluster_connection, girder_token, parent, path, assetstore_url, assetstore_id, upload=False, include=None, exclude=None): """ Download a given path on a cluster into an assetstore. :params cluster_connection: The cluster connection to access the cluster. :params girder_token: The Girder token to use to access Girder. :params parent: The target folder to import the path into. :params path: The path on the cluster to download. :params assetstore_url: The url for the assetstore to use for the import. :params assetstore_id: The id of the asseststore to import into. :params upload: Indicate if the import should upload the file data or just the metadata, the default is False. :params include: List of include regexs :params exclude: List of exclude regexs, """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token _import_path(cluster_connection, girder_client, parent, path, assetstore_url, assetstore_id, upload=upload, include=include, exclude=exclude)
class GirderClient(object): def __init__(self): token_obj = get_oc_token_obj() url = token_obj.get('apiUrl') api_key = token_obj.get('apiKey') url = os.environ.get('OC_API_URL', url) internal_url = os.environ.get('OC_INTERNAL_API_URL', url) api_key = os.environ.get('OC_API_KEY', api_key) token = os.environ.get('GIRDER_TOKEN') self.client = None self.url = url self.internal_url = internal_url if internal_url is not None: self.client = GC(apiUrl=internal_url) if api_key is not None: self.client.authenticate(apiKey=api_key) elif token is not None: self.client.token = token def __getattr__(self, name): return getattr(self.client, name)
def __init__(self, username, password, dryrun, blacklist, host=None, port=None, apiRoot=None, scheme=None): """initialization function to create a GirderCli instance, will attempt to authenticate with the designated Girder instance. :param username: username to authenticate to Girder instance. :param password: password to authenticate to Girder instance, leave this blank to be prompted. :param dryrun: boolean indicating whether to run the command or just perform a dryrun showing which files and folders will be uploaded. :param blacklist: list of filenames which will be ignored by upload. :param host: host used to connect to Girder instance, defaults to 'localhost' :param port: port used to connect to Girder instance, default is 80 for http: and 443 for https: :param apiRoot: The path on the server corresponding to the root of the Girder REST API. If None is passed, assumes '/api/v1'. :param scheme: scheme used to connect to Girder instance, defaults to 'http' """ GirderClient.__init__(self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist) interactive = password is None self.authenticate(username, password, interactive=interactive)
def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=JobStatus.SUCCESS): """Wait for all worker jobs to complete""" start_time = time.time() incompleteJobs = [] while True and (time.time() - start_time < max_wait_timeout): incompleteJobs = client.get( 'job', parameters={ # https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py # https://github.com/girder/girder_worker/blob/master/girder_worker/girder_plugin/status.py 'statuses': json.dumps([0, 1, 2, 820, 821, 822, 823, 824]), }, ) if len(incompleteJobs) == 0: break time.sleep(1) if len(incompleteJobs) > 0: raise Exception("Jobs were still running after timeout") # Verify that all jobs succeeded time.sleep(1) lastJob = client.get( 'job', parameters={ 'limit': 1, }, ) if len(lastJob) > 0 and lastJob[0]['status'] != expected_status: raise Exception(f"Some jobs did not meet their expected status: {expected_status}")
def main(): parser = argparse.ArgumentParser( description='Import analyses into minerva') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--scheme', required=False, default='http') parser.add_argument('--host', required=False, default='localhost') parser.add_argument('--port', required=False, default='8080') parser.add_argument('--api-root', required=False, default='/api/v1', help='path to the Girder REST API') parser.add_argument('--path', required=True, help='the path to import the analyses from') config = parser.parse_args() client = GirderClient(host=config.host, port=config.port, apiRoot=config.api_root, scheme=config.scheme) client.authenticate(config.username, config.password) import_analyses(client, config.path)
def __init__(self, username, password, dryrun, blacklist, host=None, port=None, apiRoot=None, scheme=None, apiUrl=None, apiKey=None): """initialization function to create a GirderCli instance, will attempt to authenticate with the designated Girder instance. Aside from username and password, all other kwargs are passed directly through to the :py:class:`girder_client.GirderClient` base class constructor. :param username: username to authenticate to Girder instance. :param password: password to authenticate to Girder instance, leave this blank to be prompted. """ GirderClient.__init__(self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist, apiUrl=apiUrl) interactive = password is None if apiKey: self.authenticate(apiKey=apiKey) elif username: self.authenticate(username, password, interactive=interactive)
def _update_girder(taskflow, body): taskflow = to_taskflow(taskflow) taskflow_id = taskflow['id'] girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token client = _create_girder_client(girder_api_url, girder_token) # If this is a retry then we have already create a task get it from # the current tasks headers. if body['retries'] > 0: taskflow_task_id = \ current_task.request.headers[TASKFLOW_TASK_ID_HEADER] # Celery always fires the postrun handler with a state of SUCCESS # for retries. So we need to save the retries here so we can # determine in the postrun handler if the task is really complete. current_task.request.headers[TASKFLOW_RETRY_HEADER] \ = body['retries'] else: # This is a new task so create a taskflow task instance body = { 'celeryTaskId': body['id'], 'name': body['task'] } url = 'taskflows/%s/tasks' % taskflow_id r = client.post(url, data=json.dumps(body)) taskflow_task_id = r['_id'] return taskflow, taskflow_task_id
def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post('https://newt.nersc.gov/newt/auth', { 'username': self._girder_user, 'password': self._girder_password }) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()['newt_sessionid'] # Now authenticate with Girder using the session id url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') self._user_id = user['_id'] r = self._client.listFolder(self._user_id, 'user', name='Private') r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id']
def _update_girder(taskflow, body): taskflow = to_taskflow(taskflow) taskflow_id = taskflow['id'] girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token client = _create_girder_client(girder_api_url, girder_token) # If this is a retry then we have already create a task get it from # the current tasks headers. if body['retries'] > 0: taskflow_task_id = \ current_task.request.headers[TASKFLOW_TASK_ID_HEADER] # Celery always fires the postrun handler with a state of SUCCESS # for retries. So we need to save the retries here so we can # determine in the postrun handler if the task is really complete. current_task.request.headers[TASKFLOW_RETRY_HEADER] \ = body['retries'] else: # This is a new task so create a taskflow task instance body = {'celeryTaskId': body['id'], 'name': body['task']} url = 'taskflows/%s/tasks' % taskflow_id r = client.post(url, data=json.dumps(body)) taskflow_task_id = r['_id'] return taskflow, taskflow_task_id
def test_sanity_checks(): gc = GirderClient(apiUrl=source_api_root) resp = gc.get('/', jsonResp=False) assert resp.status_code == 200 assert resp.headers['Content-Type'] == 'text/html;charset=utf-8' resp = gc.get('system/check', jsonResp=False) assert resp.status_code == 200
def test_upgrade_pipelines(admin_client: GirderClient): cnf = admin_client.get('dive_configuration/pipelines') if 'detector' not in cnf: admin_client.post( 'dive_configuration/upgrade_pipelines', data=json.dumps(tasks.UPGRADE_JOB_DEFAULT_URLS), ) wait_for_jobs(admin_client, 1000)
def _taskflow_task_finished(taskflow, taskflow_task_id): girder_token = taskflow['girder_token'] girder_api_url = taskflow['girder_api_url'] client = GirderClient(apiUrl=girder_api_url) client.token = girder_token url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id) return client.put(url)
def _importAnalysis(self): """Setup and import analyses for bsve tests.""" if self._import_done: return path = "/minerva_analysis/folder" response = self.request(path=path, method="POST", user=self._user) self.assertStatusOk(response) analyses_folder = response.json["folder"] # import the bsve analysis client = GirderClient("localhost", girder_port) client.authenticate("minervauser", "password") bsve_analysis_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../analyses/bsve") ) import_analyses.import_analyses(client, bsve_analysis_path) path = "/item" params = {"folderId": analyses_folder["_id"]} response = self.request(path=path, method="GET", params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 2, "Expecting only one analysis") for analysis in response.json: if analysis["name"] == "bsve search": search_analysis = analysis elif analysis["name"] == "MMWR data import": soda_analysis = analysis else: self.fail('Unexpected analysis found "%s".' % analysis["name"]) expected_meta = { u"minerva": { u"analysis_type": u"bsve_search", u"analysis_name": u"bsve search", u"analysis_id": search_analysis["_id"], } } self.assertEquals(search_analysis["meta"], expected_meta, "Unexpected value for search meta data") expected_meta = { u"minerva": { u"analysis_type": u"mmwr_import_data", u"analysis_name": u"MMWR data import", u"analysis_id": soda_analysis["_id"], } } self.assertEquals(soda_analysis["meta"], expected_meta, "Unexpected value for soda meta data") # create the dataset folder path = "/minerva_dataset/folder" params = {"userId": self._user["_id"]} response = self.request(path=path, method="POST", params=params, user=self._user) self.assertStatusOk(response) self._importDone = True
def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id']
def main(ctx, api_key, api_url): """Openchemistry Client The client can be used to fetch molecules, add molecules, etc. """ gc = GirderClient(apiUrl=api_url) if api_key is not None: gc.authenticate(apiKey=api_key) ctx.obj = gc
def import_calc(config): try: target_port = None if config.port: target_port = config.port target_scheme = None if config.scheme: target_scheme = config.scheme target_apiroot = None if config.apiroot: target_apiroot = config.apiroot client = GirderClient(host=config.host, port=target_port, scheme=target_scheme, apiRoot=target_apiroot) client.authenticate(apiKey=config.apiKey) me = client.get('/user/me') if not me: print('Error: Girder token invalid, please verify') return folderParams = { 'parentId': me['_id'], 'parentType': 'user', 'name': 'Private' } # Get the private folder id first folder = next(client.listResource('folder', folderParams)) folder = next(client.listFolder(me['_id'], 'user', 'Private')) for file_name in config.datafile: print ('\nUploading ' + file_name) file_id = {} with open(file_name, 'r') as fp: fileNameBase = os.path.basename(file_name) size = os.path.getsize(file_name) file_id = client.uploadFile(folder['_id'], fp, fileNameBase, size, 'folder') body = { 'fileId': file_id['_id'] } if config.public: body['public'] = True mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body)) if mol and '_id' in mol: config.moleculeId = mol['_id'] print('Molecule ID: ' + mol['_id']) else: print(mol) except HttpError as error: print(error.responseText, file=sys.stderr)
def test_user_creation(admin_client: GirderClient, user: dict): try: admin_client.createUser( user['login'], user['email'], user['firstName'], user['lastName'], user['password'], ) except HttpError as err: if err.response.json()['message'] != 'That login is already registered.': raise err
def gc_init(api_url='https://girder.hub.yt/api/v1', api_key=None): from girder_client import GirderClient if api_key is None: try: api_key = os.environ['GIRDER_API_KEY'] except KeyError as err: msg = 'please define GIRDER_API_KEY environment variable\n' msg += ' or pass api_key kwargs.' raise KeyError(msg) gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) return gc
def _ingest(project, composite, dir, channel_map, api_url, api_key): if dir[-1] != '/': dir += '/' gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) channel_map = json.load(channel_map) channel_map = {channel.upper():element.lower() for (channel,element) in channel_map.items()} experiments = _ingest_runs(gc, project, composite, dir) #(run_ids, runs) = _ingest_runs(gc, project, composite, dir) _ingest_samples(gc, project, composite, dir, experiments, channel_map)
def upload_file(cluster, girder_token, file, path): """ Upload a file to a cluster :param cluster: The cluster to upload to. :param girder_tokebn: The Grider token for Girder access. :param file: The Girder file object. :param path: The path on the cluster to upload to. """ girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token with get_connection(girder_token, cluster) as conn: conn.makedirs(os.path.dirname(path)) _upload_file(conn, girder_client, file, path)
def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post('https://newt.nersc.gov/newt/auth', { 'username': self._girder_user, 'password': self._girder_password}) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()['newt_sessionid'] # Now authenticate with Girder using the session id url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') self._user_id = user['_id'] r = self._client.listFolder(self._user_id, 'user', name='Private') self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id']
def testClientMetadataExtractor(testData, user): item = Item().load(testData['item']['_id'], user=user) assert item['name'] == testData['name'] del item['meta'] item = Item().save(item) assert 'meta' not in item client = GirderClient('localhost', int(os.environ['GIRDER_PORT'])) client.authenticate(user['login'], 'password') extractor = ClientMetadataExtractor(client, testData['path'], testData['item']['_id']) extractor.extractMetadata() item = Item().load(testData['item']['_id'], user=user) assert item['name'] == testData['name'] assert 'meta' in item assert item['meta']['MIME type'] == testData['mimeType']
def __init__(self, username, password, dryrun, blacklist, host=None, port=None, apiRoot=None, scheme=None, apiUrl=None): """initialization function to create a GirderCli instance, will attempt to authenticate with the designated Girder instance. Aside from username and password, all other kwargs are passed directly through to the :py:class:`girder_client.GirderClient` base class constructor. :param username: username to authenticate to Girder instance. :param password: password to authenticate to Girder instance, leave this blank to be prompted. """ GirderClient.__init__(self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist, apiUrl=apiUrl) interactive = password is None self.authenticate(username, password, interactive=interactive)
def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post( "https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password} ) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()["newt_sessionid"] # Now authenticate with Girder using the session id url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1" % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies["girderToken"] user = self._client.get("user/me") self._user_id = user["_id"] r = self._client.listFolder(self._user_id, "user", name="Private") r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]["_id"]
def init_girder(api_key=None, api_url='https://girder.hub.yt/api/v1'): """Initialize girder client, rely on environment variable: GIRDER_API_KEY, connect to yt Hub. Args: api_key (str, optional): use GIRDER_API_KEY env. var. by default api_url (str, optional): use yt Hub v1 by default Return: GirderClient: initialized girder client """ from girder_client import GirderClient if api_key is None: import os api_key = os.environ['GIRDER_API_KEY'] gc = GirderClient(apiUrl=api_url) gc.authenticate(apiKey=api_key) return gc
def main(): parser = argparse.ArgumentParser(description='Import analyses into minerva') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--scheme', required=False, default='http') parser.add_argument('--host', required=False, default='localhost') parser.add_argument('--port', required=False, default='8080') parser.add_argument('--api-root', required=False, default='/api/v1', help='path to the Girder REST API') parser.add_argument('--path', required=True, help='the path to import the analyses from') config = parser.parse_args() client = GirderClient(host=config.host, port=config.port, apiRoot=config.api_root, scheme=config.scheme) client.authenticate(config.username, config.password) import_analyses(client, config.path)
def testClientMetadataExtractor(self): item = self.model('item').load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertNotHasKeys(item, ['meta']) clientPath = os.path.join(ROOT_DIR, 'clients', 'python') sys.path.insert(0, clientPath) from girder_client import GirderClient client = GirderClient('localhost', int(os.environ['GIRDER_PORT'])) client.authenticate(self.user['login'], self.password) extractor = ClientMetadataExtractor(client, self.path, self.item['_id']) extractor.extractMetadata() sys.path.remove(clientPath) item = self.model('item').load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertHasKeys(item, ['meta']) self.assertEqual(item['meta']['MIME type'], self.mimeType)
def testClientMetadataExtractor(self): item = Item().load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertNotHasKeys(item, ['meta']) clientPath = os.path.join(ROOT_DIR, 'clients', 'python') sys.path.insert(0, clientPath) from girder_client import GirderClient client = GirderClient('localhost', int(os.environ['GIRDER_PORT'])) client.authenticate(self.user['login'], self.password) extractor = ClientMetadataExtractor(client, self.path, self.item['_id']) extractor.extractMetadata() sys.path.remove(clientPath) item = Item().load(self.item['_id'], user=self.user) self.assertEqual(item['name'], self.name) self.assertHasKeys(item, ['meta']) self.assertEqual(item['meta']['MIME type'], self.mimeType)
def upload_zipped_flat_media_files( gc: GirderClient, manager: JobManager, folderId: str, working_directory: Path, create_subfolder=False, ): """Takes a flat folder of media files and/or annotation and generates a dataset from it""" listOfFileNames = os.listdir(working_directory) validation = gc.sendRestRequest('POST', '/dive_dataset/validate_files', json=listOfFileNames) root_folderId = folderId default_fps = gc.getFolder(root_folderId).get( f"meta.{constants.FPSMarker}", -1) if validation.get('ok', False): manager.write(f"Annotations: {validation['annotations']}\n") manager.write(f"Media: {validation['media']}\n") dataset_type = validation['type'] manager.write(f"Type: {dataset_type}\n") if create_subfolder != '': sub_folder = gc.createFolder( folderId, create_subfolder, reuseExisting=True, ) root_folderId = str(sub_folder["_id"]) # Upload all resulting items back into the root folder manager.updateStatus(JobStatus.PUSHING_OUTPUT) # create a source folder to place the zipFile inside of gc.upload(f'{working_directory}/*', root_folderId) if dataset_type == constants.ImageSequenceType and default_fps == -1: default_fps = 1 gc.addMetadataToFolder( str(root_folderId), { constants.TypeMarker: dataset_type, constants.FPSMarker: default_fps }, ) # After uploading the default files we do a the postprocess for video conversion now gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(root_folderId)}") else: manager.write(f"Message: {validation['message']}\n") raise Exception("Could not Validate media Files")
def main(config): client = GirderClient(apiUrl=config.girder_api_url) client.authenticate(config.girder_user, config.girder_password) # Load any parameters params = {} if config.taskflow_start_params is not None: with open(config.taskflow_start_params) as fp: params = json.load(fp) print params try: print ('Running %s taskflow ...' % config.taskflow_start_params) taskflow_id = create_taskflow( client, config.taskflow_class) # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url, data=json.dumps(params)) # Wait for it to complete wait_for_complete(client, taskflow_id) except HttpError as ex: print( ex.responseText)
def get_girder_client() -> GirderClient: client = GirderClient(apiUrl="https://viame.kitware.com/api/v1") apikey = os.environ.get('GIRDER_API_KEY', None) if apikey: client.authenticate(apiKey=apikey) else: client.authenticate(interactive=True) return client
def __init__(self, username, password, dryrun, blacklist, host="localhost", port=8080, apiRoot=None, scheme="http"): """initialization function to create a GirderCli instance, will attempt to authenticate with the designated Girder instance. :param username: username to authenticate to Girder instance. :param password: password to authenticate to Girder instance, leave this blank to be prompted. :param dryrun: boolean indicating whether to run the command or just perform a dryrun showing which files and folders will be uploaded. :param blacklist: list of filenames which will be ignored by upload. :param host: host used to connect to Girder instance, defaults to 'localhost' :param port: port used to connect to Girder instance, defaults to 8080 :param apiRoot: The path on the server corresponding to the root of the Girder REST API. If None is passed, assumes '/api/v1'. :param scheme: scheme used to connect to Girder instance, defaults to 'http'; if passing 'https' port should likely be 443. """ GirderClient.__init__( self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist ) interactive = password is None self.authenticate(username, password, interactive=interactive)
def test_extract_download(data): fileId, filename = data filepath = localDataRoot / str(filename) if not filepath.exists(): gc = GirderClient(apiUrl=source_api_root) gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY')) gc.downloadFile(fileId, str(filepath)) with zipfile.ZipFile(filepath, 'r') as zipref: zipref.extractall(localDataRoot)
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('-c', default='remote', choices=['remote', 'direct'], help='command to run') parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) else: print('No implementation for command %s' % args.c)
def main(): """Create the folder hierarchy with metadata in a Girder instance.""" args = parser.parse_args() g = GirderClient(host=args.host, port=args.port, scheme=args.scheme) g.authenticate(args.username, args.password) def create_folder_on_demand(parent_folder_id, folder_name): existing_folders = list( g.listFolder(parent_folder_id, name=folder_name)) if not len(existing_folders): sought_folder = g.createFolder(parent_folder_id, name=folder_name) else: sought_folder = existing_folders[0] return sought_folder metadata_file = 'metadata.json' with open(metadata_file) as json_file: metadata = json.load(json_file) parent_folder_id = args.parent_folder_id for subject_id, subject_metadata in metadata.items(): subject_folder = create_folder_on_demand(parent_folder_id, subject_id) for (scan_time, scan_date, scan_weight) in subject_metadata['scans']: create_folder_on_demand(subject_folder['_id'], scan_time)
def upload_benchmark_results(benchmark_bin, api_key=None): hostname = socket.gethostname().lower() results_dir = os.path.join(benchmark_bin, 'BenchmarkResults', hostname) if not os.path.exists(results_dir): sys.stderr.write('Expected results directory does not exist: ' + results_dir) sys.exit(1) from girder_client import GirderClient gc = GirderClient(apiUrl='https://data.kitware.com/api/v1') gc.authenticate(apiKey=api_key) # ITK/PerformanceBenchmarkingResults folder_id = '5af50c818d777f06857985e3' hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder') gc.upload(os.path.join(results_dir, '*.json'), hostname_folder['_id'], leafFoldersAsItems=False, reuseExisting=True)
def gw_task_prerun(task=None, sender=None, task_id=None, args=None, kwargs=None, **rest): """Deserialize the jobInfoSpec passed in through the headers. This provides the a JobManager class as an attribute of the task before task execution. decorated functions may bind to their task and have access to the job_manager for logging and updating their status in girder. """ if is_builtin_celery_task(sender.name): return try: task.job_manager = _job_manager(task.request, task.request.headers) _update_status(task, JobStatus.RUNNING) except JobSpecNotFound: task.job_manager = None logger.warn('No jobInfoSpec. Setting job_manager to None.') except StateTransitionException: # Fetch the current status of the job status = task.job_manager.refreshStatus() # If we are canceling we want to stay in that state if status != JobStatus.CANCELING: raise try: task.girder_client = GirderClient(apiUrl=task.request.girder_api_url) task.girder_client.token = task.request.girder_client_token except AttributeError: task.girder_client = None # Deserialize girder_result_hooks if they exist if hasattr(task.request, 'girder_result_hooks'): u = jsonpickle.unpickler.Unpickler() task.request.girder_result_hooks = \ [u.restore(grh) for grh in task.request.girder_result_hooks]
def testImportAnalyses(self): """ Test importing a romanesco analysis """ client = GirderClient('localhost', girder_port) client.authenticate(self._username, self._password) path = os.path.dirname(os.path.realpath(__file__)) analyses_path = os.path.join(path, 'analyses') import_analyses.import_analyses(client, analyses_path) # Get the analysis folder path = '/minerva_analysis/folder' response = self.request(path=path, method='GET', params={}, user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] path = '/item' params = { 'folderId': analyses_folder['_id'] } response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 2, 'Expecting two analyses') analysis = response.json[0] self.assertEquals(analysis['name'], 'add', 'Expecting analysis one name to be "add"') expected_meta = { u'minerva': { u'analysis_type': u'add', u'analysis_name': u'add', u'analysis_id': analysis['_id'] }, u'analysis': { u'inputs': [{ u'default': { u'data': u'0', u'format': u'json' }, u'type': u'number', u'name': u'a', u'format': u'number' }, { u'type': u'number', u'name': u'b', u'format': u'number' }], u'script': u'c = a + b', u'mode': u'python', u'outputs': [{ u'type': u'number', u'name': u'c', u'format': u'number' }], u'name': u'add' } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data') analysis = response.json[1] self.assertEquals(analysis['name'], 'local', 'Expecting analysis two name to be "local"') expected_meta = { u'minerva': { u'analysis_type': u'local type', u'analysis_name': u'local', u'analysis_id': analysis['_id'] } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
def _create_girder_client(girder_api_url, girder_token): client = GirderClient(apiUrl=girder_api_url) client.token = girder_token return client
from girder_client import GirderClient import json import os import pymongo import sys if len(sys.argv) < 2: print "%s /path/to/arborCollections" % sys.argv[0] print "This should be the root of a directory tree containing .json analysis files" sys.exit(1) arbor_collections_path = sys.argv[1] # Authenticate with Girder. c = GirderClient(host='localhost', port=8080) c.authenticate('girder', 'girder') # Recursively search the specified directory for .json files. for root, dirs, files in os.walk(arbor_collections_path): for file in files: if not file.endswith(".json"): continue # Get the name of this file and the directory that it's in. # We use the directory name as the collection name in Girder. fullpath = os.path.join(root, file) analysis_filename = os.path.basename(fullpath) analysis_name = os.path.splitext(analysis_filename)[0] analysis_dir = os.path.dirname(fullpath) collection_name = os.path.basename(analysis_dir) # Create this collection if it doesn't already exist.
parser = ArgumentParser(description='Initialize the girder environment') parser.add_argument('--user', help='name of the user to create') parser.add_argument('--password', help='password of the user to create') parser.add_argument('--host', help='host to connect to') parser.add_argument('--port', type=int, help='port to connect to') parser.add_argument('--data-root', help=("root directory for the girder " "server's local filesystem data")) parser.add_argument('--broker', help='romanesco broker URI') parser.add_argument('--hdfs-user', help='Hadoop HDFS user') parser.add_argument('--hdfs-namenode', help='Hadoop HDFS namenode') parser.add_argument('--hdfs-port', type=int, help='Hadoop HDFS port') parser.add_argument('--web-hdfs-port', type=int, help='Hadoop WebHDFS port') args = parser.parse_args() client = GirderClient(host=args.host, port=args.port) register_and_authenticate(client, login=args.user, password=args.password, email='{}@localhost.com'.format(args.user), firstName='Girder', lastName='Admin') local_assetstore_name = 'local' if find_assetstore(local_assetstore_name) is None: client.post('assetstore', parameters=dict(name=local_assetstore_name, type=str(AssetstoreType.FILESYSTEM), root=os.path.join(args.data_root, 'assetstores',
class BaseIntegrationTest(unittest.TestCase): def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True): super(BaseIntegrationTest, self).__init__(name) self._job_id = None self._script_id = None self._output_folder_id = None self._input_folder_id = None self._girder_url = girder_url self._girder_user = girder_user self._girder_password = girder_password self._job_timeout = job_timeout self._data = 'Need more input!' self._cleanup = cleanup def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): if not self._cleanup: return if self._job_id: try: url = 'jobs/%s' % self._job_id self._client.delete(url) except Exception as e: traceback.print_exc() if self._script_id: try: url = 'scripts/%s' % self._script_id self._client.delete(url) except Exception: traceback.print_exc() if self._output_folder_id: try: url = 'folder/%s' % self._output_folder_id self._client.delete(url) except Exception: traceback.print_exc() if self._input_folder_id: try: url = 'folder/%s' % self._input_folder_id self._client.delete(url) except Exception: traceback.print_exc() def create_script(self, commands=[ 'sleep 10', 'cat CumulusIntegrationTestInput' ]): body = { 'commands': commands, 'name': 'CumulusIntegrationTestLob' } r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] def create_input(self, folder_name='CumulusInput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._input_folder_id = r['_id'] size = len(self._data) item = self._client.uploadFile(self._input_folder_id, StringIO(self._data), 'CumulusIntegrationTestInput', size, parentType='folder') self._item_id = item['itemId'] def create_output_folder(self, folder_name='CumulusOutput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._output_folder_id = r['_id'] def create_job(self, job_name='CumulusIntegrationTestJob', tail=None): body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [ { 'folderId': self._input_folder_id, 'path': '.' } ] } if tail: body['output'].append({ "path": tail, "tail": True }) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] def submit_job(self, job_params={}, timeout=None): url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(job_params)) start = time.time() while True: time.sleep(1) r = self._client.get('jobs/%s' % self._job_id) if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) self.fail(str(r)) elif r['status'] == 'complete': break if time.time() - start > timeout: self.fail('Job didn\'t complete in timeout') def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) path = os.path.join(tempfile.gettempdir(), self._job_id) try: self._client.downloadFile(r[0]['_id'], path) with open(path, 'rb') as fp: self.assertEqual(fp.read(), self._data) finally: if os.path.exists(path): os.remove(path)
@click.group(context_settings=_CONTEXT_SETTINGS, cls=_Group) @click.option('--api-url', default=None, help='RESTful API URL ' '(e.g https://girder.example.com:443/%s)' % GirderClient.DEFAULT_API_ROOT) @click.option('--api-key', envvar='GIRDER_API_KEY', default=None, help='[default: GIRDER_API_KEY env. variable]') @click.option('--username', default=None) @click.option('--password', default=None) # Advanced options @click.option('--host', default=None, cls=_AdvancedOption, help="[default: %s]" % GirderClient.DEFAULT_HOST) @click.option('--scheme', default=None, cls=_AdvancedOption, help="[default: %s if %s else %s]" % ( GirderClient.getDefaultScheme(GirderClient.DEFAULT_HOST), GirderClient.DEFAULT_HOST, GirderClient.getDefaultScheme("girder.example.com"))) @click.option('--port', default=None, cls=_AdvancedOption, help="[default: %s if %s; %s if %s else %s]" % ( GirderClient.DEFAULT_HTTPS_PORT, "https", GirderClient.DEFAULT_LOCALHOST_PORT, "localhost", GirderClient.DEFAULT_HTTP_PORT, )) @click.option('--api-root', default=None, help='relative path to the Girder REST API ' '[default: %s]' % GirderClient.DEFAULT_API_ROOT, show_default=True, cls=_AdvancedOption) @click.version_option(version=__version__, prog_name='Girder command line interface')
offset += limit return result parser = ArgumentParser(description='Initialize the girder environment') parser.add_argument('--admin', help='name:pass for the admin user') parser.add_argument('--host', help='host to connect to') parser.add_argument('--port', type=int, help='port to connect to') parser.add_argument('--broker', help='girder worker broker URI') parser.add_argument('--s3', help='name of S3 bucket') parser.add_argument('--aws-key-id', help='aws key id') parser.add_argument('--aws-secret-key', help='aws secret key') args = parser.parse_args() client = GirderClient(host=args.host, port=args.port) user, password = args.admin.split(":", 1) if find_user('girder'): client.authenticate('girder', 'girder') ensure_user(client, login=user, password=password, email='*****@*****.**', firstName='Girder', lastName='Admin') client.authenticate(user, password)
def testBsveSearchAnalysis(self): # create the analysis folder path = '/minerva_analysis/folder' response = self.request(path=path, method='POST', user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] # import the bsve analysis client = GirderClient('localhost', girder_port) client.authenticate('minervauser', 'password') bsve_analysis_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve')) import_analyses.import_analyses(client, bsve_analysis_path) path = '/item' params = { 'folderId': analyses_folder['_id'] } response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 1, 'Expecting only one analysis') analysis = response.json[0] self.assertEquals(analysis['name'], 'bsve search', 'Expecting analysis name to be "bsve search"') expected_meta = { u'minerva': { u'analysis_type': u'bsve_search', u'analysis_name': u'bsve search', u'analysis_id': analysis['_id'] } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data') # create the dataset folder path = '/minerva_dataset/folder' params = { 'userId': self._user['_id'], } response = self.request(path=path, method='POST', params=params, user=self._user) self.assertStatusOk(response) # mock the calls to bsve search @urlmatch(netloc=r'(.*\.)?beta-search.bsvecosystem.net(.*)$') def bsve_mock(url, request): if url.path.split('/')[-1] == 'request': return httmock.response(200, '12345') else: pluginTestDir = os.path.dirname(os.path.realpath(__file__)) filepath = os.path.join(pluginTestDir, 'data', 'bsve_search.json') with open(filepath) as bsve_search_file: content = { 'status': 1, 'results': json.load(bsve_search_file) } headers = { 'content-length': len(content), 'content-type': 'application/json' } return httmock.response(200, content, headers, request=request) with HTTMock(bsve_mock): response = self.request( path='/minerva_analysis/bsve_search', method='POST', params={ 'datasetName': 'test dataset', 'bsveSearchParams': '{}' }, user=self._user ) # wait for the async job to complete searchResultsFinished = False count = 0 while not searchResultsFinished and count < 5: # get the dataset and check if it has been updated path = '/minerva_dataset/%s/dataset' % str(response.json['dataset_id']) response = self.request( path=path, method='GET', user=self._user ) dataset = response.json if 'json_row' in dataset: searchResultsFinished = True else: time.sleep(2) count += 1 # ensure the first row of results was added to the dataset self.assertTrue('json_row' in dataset, 'json_row expected in dataset') self.assertTrue('data' in dataset['json_row'], 'data should be in json_row') self.assertTrue('Longitude' in dataset['json_row']['data'], 'data.Longitude should be in json_row') # ensure that we can map the Lat/Long to geojson, as this json has # unicode values for Lat/Long # update the minerva metadata with coordinate mapping metadata = {'minerva': dataset} metadata['minerva']['mapper'] = { "latitudeKeypath": "data.Latitude", "longitudeKeypath": "data.Longitude" } path = '/item/{}/metadata'.format(dataset['dataset_id']) response = self.request( path=path, method='PUT', user=self._user, body=json.dumps(metadata), type='application/json' ) metadata = response.json # create geojson in the dataset path = '/minerva_dataset/{}/geojson'.format(dataset['dataset_id']) response = self.request( path=path, method='POST', user=self._user, ) self.assertHasKeys(response.json, ['geojson_file'])
def main(config): client = GirderClient(apiUrl=config.girder_api_url) client.authenticate(config.girder_user, config.girder_password) try: # First run the simple flow print ('Running simple taskflow ...') taskflow_id = create_taskflow( client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow') # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url) # Wait for it to complete wait_for_complete(client, taskflow_id) # First run the simple flow print ('Running linked taskflow ...') taskflow_id = create_taskflow( client, 'cumulus.taskflow.core.test.mytaskflows.LinkTaskFlow') # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url) # Wait for it to complete wait_for_complete(client, taskflow_id) # Test terminating a simple flow print ('Running simple taskflow ...') taskflow_id = create_taskflow( client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow') # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url) time.sleep(4) print ('Terminate the taskflow') url = 'taskflows/%s/terminate' % (taskflow_id) client.put(url) # Wait for it to terminate wait_for_terminated(client, taskflow_id) # Now delete it print ('Delete the taskflow') url = 'taskflows/%s' % (taskflow_id) try: client.delete(url) except HttpError as ex: if ex.status != 202: raise # Wait for it to terminate wait_for_deletion(client, taskflow_id) # Now try something with a chord print ('Running taskflow containing a chord ...') taskflow_id = create_taskflow( client, 'cumulus.taskflow.core.test.mytaskflows.ChordTaskFlow') # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url) # Wait for it to complete wait_for_complete(client, taskflow_id) # Now try a workflow that is the two connected together print ('Running taskflow that connects to parts together ...') taskflow_id = create_taskflow( client, 'cumulus.taskflow.core.test.mytaskflows.ConnectTwoTaskFlow') # Start the task flow url = 'taskflows/%s/start' % (taskflow_id) client.put(url) # Wait for it to complete wait_for_complete(client, taskflow_id) # # Now try a composite workflow approach ... # print ('Running taskflow that is a composite ...') # taskflow_id = create_taskflow( # client, 'cumulus.taskflow.core.test.mytaskflows.MyCompositeTaskFlow') # # # Start the task flow # url = 'taskflows/%s/start' % (taskflow_id) # client.put(url) # # # Wait for it to complete # wait_for_complete(client, taskflow_id) except HttpError as ex: print( ex.responseText)
import sys parser = argparse.ArgumentParser() parser.add_argument("path", type=str, help="path to Arbor web apps") parser.add_argument("-g", "--girder-host", type=str, default='localhost', help="host to Girder instance") parser.add_argument("-p", "--girder-port", type=int, default=9000, help="port to Girder instance") args = parser.parse_args() # Get the ID for our Analyses folder. c = GirderClient(host=args.girder_host, port=args.girder_port) c.authenticate('girder', 'girder') folderSearch = c.get('resource/search', parameters={ 'q': 'Analyses', 'types': '["folder"]' }) folderId = folderSearch['folder'][0]['_id'] # Disable authorization requirements for running romanesco tasks c.put('system/setting', parameters={ 'key': 'flow.require_auth', 'value': 'false' }) # Check if these analyses already exist. If so, we won't re-upload them. uploadACR = False
def upload_path(cluster_connection, girder_token, folder_id, path): girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl) girder_client.token = girder_token cluster_connection.makedirs(path) _upload_path(cluster_connection, girder_client, folder_id, path)