class NewtIntegrationTest(BaseIntegrationTest): def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60 * 5): super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout) self._cluster_id = None self._machine = machine def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post( "https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password} ) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()["newt_sessionid"] # Now authenticate with Girder using the session id url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = "%s/api/v1" % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies["girderToken"] user = self._client.get("user/me") self._user_id = user["_id"] r = self._client.listFolder(self._user_id, "user", name="Private") r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]["_id"] def tearDown(self): super(NewtIntegrationTest, self).tearDown() if self._cluster_id: try: url = "clusters/%s" % self._cluster_id self._client.delete(url) except Exception: traceback.print_exc() def create_cluster(self): body = {"config": {"host": self._machine}, "name": "NewtIntegrationTest", "type": "newt"} r = self._client.post("clusters", data=json.dumps(body)) self._cluster_id = r["_id"] # Now test the connection r = self._client.put("clusters/%s/start" % self._cluster_id) sleeps = 0 while True: time.sleep(1) r = self._client.get("clusters/%s/status" % self._cluster_id) if r["status"] == "running": break elif r["status"] == "error": r = self._client.get("clusters/%s/log" % self._cluster_id) self.fail(str(r)) if sleeps > 9: self.fail("Cluster never moved into running state") sleeps += 1 def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i["name"].startswith("CumulusIntegrationTestJob-%s.o" % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get("item/%s/files" % i["_id"]) self.assertEqual(len(r), 1) url = "%s/api/v1/file/%s/download" % (self._girder_url, r[0]["_id"]) r = self._session.get(url) self.assertEqual(r.content, self._data) def test(self): try: self.create_cluster() self.create_script() self.create_input() self.create_output_folder() self.create_job() self.submit_job(timeout=self._job_timeout) self.assert_output() except HttpError as error: self.fail(error.responseText)
def test_sanity_checks(): gc = GirderClient(apiUrl=source_api_root) resp = gc.get('/', jsonResp=False) assert resp.status_code == 200 assert resp.headers['Content-Type'] == 'text/html;charset=utf-8' resp = gc.get('system/check', jsonResp=False) assert resp.status_code == 200
def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=JobStatus.SUCCESS): """Wait for all worker jobs to complete""" start_time = time.time() incompleteJobs = [] while True and (time.time() - start_time < max_wait_timeout): incompleteJobs = client.get( 'job', parameters={ # https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py # https://github.com/girder/girder_worker/blob/master/girder_worker/girder_plugin/status.py 'statuses': json.dumps([0, 1, 2, 820, 821, 822, 823, 824]), }, ) if len(incompleteJobs) == 0: break time.sleep(1) if len(incompleteJobs) > 0: raise Exception("Jobs were still running after timeout") # Verify that all jobs succeeded time.sleep(1) lastJob = client.get( 'job', parameters={ 'limit': 1, }, ) if len(lastJob) > 0 and lastJob[0]['status'] != expected_status: raise Exception(f"Some jobs did not meet their expected status: {expected_status}")
def test_upgrade_pipelines(admin_client: GirderClient): cnf = admin_client.get('dive_configuration/pipelines') if 'detector' not in cnf: admin_client.post( 'dive_configuration/upgrade_pipelines', data=json.dumps(tasks.UPGRADE_JOB_DEFAULT_URLS), ) wait_for_jobs(admin_client, 1000)
def import_calc(config): try: target_port = None if config.port: target_port = config.port target_scheme = None if config.scheme: target_scheme = config.scheme target_apiroot = None if config.apiroot: target_apiroot = config.apiroot client = GirderClient(host=config.host, port=target_port, scheme=target_scheme, apiRoot=target_apiroot) client.authenticate(apiKey=config.apiKey) me = client.get('/user/me') if not me: print('Error: Girder token invalid, please verify') return folderParams = { 'parentId': me['_id'], 'parentType': 'user', 'name': 'Private' } # Get the private folder id first folder = next(client.listResource('folder', folderParams)) folder = next(client.listFolder(me['_id'], 'user', 'Private')) for file_name in config.datafile: print ('\nUploading ' + file_name) file_id = {} with open(file_name, 'r') as fp: fileNameBase = os.path.basename(file_name) size = os.path.getsize(file_name) file_id = client.uploadFile(folder['_id'], fp, fileNameBase, size, 'folder') body = { 'fileId': file_id['_id'] } if config.public: body['public'] = True mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body)) if mol and '_id' in mol: config.moleculeId = mol['_id'] print('Molecule ID: ' + mol['_id']) else: print(mol) except HttpError as error: print(error.responseText, file=sys.stderr)
def download_source_media(girder_client: GirderClient, datasetId: str, dest: Path) -> Tuple[List[str], str]: """Download media for dataset to dest path""" media = models.DatasetSourceMedia( **girder_client.get(f'dive_dataset/{datasetId}/media')) dataset = models.GirderMetadataStatic( **girder_client.get(f'dive_dataset/{datasetId}')) if dataset.type == constants.ImageSequenceType: for frameImage in media.imageData: girder_client.downloadItem(frameImage.id, str(dest)) return [str(dest / image.filename) for image in media.imageData], dataset.type elif dataset.type == constants.VideoType and media.video is not None: destination_path = str(dest / media.video.filename) girder_client.downloadFile(media.video.id, destination_path) return [destination_path], dataset.type else: raise Exception(f"unexpected metadata {str(dataset.dict())}")
def test_reset_job_logs(admin_client: GirderClient): # remove any failed jobs. for job in admin_client.get( 'job/all', parameters={"statuses": json.dumps([0, 1, 2, 4, 5, 824])}): admin_client.delete(f'job/{job["_id"]}')
def getTestFolder(client: GirderClient): me = client.get('user/me') privateFolder = client.loadOrCreateFolder("Integration", me['_id'], 'user') return privateFolder
class CumulusClient(): '''Application interface to cumulus-based client for HPC systems supporting NEWT API. Note: the methods must be called in a specific order! create_cluster() create_omega3p_script() create_job() upload_inputs() submit_job() Then optionally: monitor_job() download_results() release_resources() ''' # --------------------------------------------------------------------- def __init__(self, girder_url, newt_sessionid): ''' ''' self._client = None self._cluster_id = None self._girder_url = girder_url self._input_folder_id = None self._job_folder_id = None self._job_id = None self._output_folder_id = None self._private_folder_id = None self._script_id = None self._session = requests.Session() # Authenticate with Girder using the newt session id url = '%s/api/v1/newt/authenticate/%s' % \ (self._girder_url, newt_sessionid) r = self._session.put(url) if r.status_code != 200: raise HttpError(r.status_code, r.text, r.url, r.request.method) # Instantiate Girder client url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') #print 'user', user user_id = user['_id'] r = self._client.listFolder(user_id, 'user', name='Private') if len(r) != 1: raise Exception('Wrong number of users; should be 1 got %s' % len(r)) self._private_folder_id = r[0]['_id'] print 'private_folder_id', self._private_folder_id # --------------------------------------------------------------------- def job_id(self): '''Returns current job id (which may be None) ''' return self._job_id # --------------------------------------------------------------------- def create_cluster(self, machine_name, cluster_name=None): ''' ''' if cluster_name is None: user = self._client.get('user/me') user_name = user.get('firstName', 'user') cluster_name = '%s.%s' % (machine_name, user_name) cluster = None cluster_list = self._client.get('clusters') for extant_cluster in cluster_list: if extant_cluster['name'] == cluster_name: cluster = extant_cluster self._cluster_id = extant_cluster['_id'] break if not cluster: body = { 'config': { 'host': machine_name }, 'name': cluster_name, 'type': 'newt' } r = self._client.post('clusters', data=json.dumps(body)) self._cluster_id = r['_id'] print 'cluster_id', self._cluster_id # Reset the state of the cluster body = {'status': 'created'} r = self._client.patch('clusters/%s' % self._cluster_id, data=json.dumps(body)) # Now test the connection r = self._client.put('clusters/%s/start' % self._cluster_id) sleeps = 0 while True: time.sleep(1) r = self._client.get('clusters/%s/status' % self._cluster_id) if r['status'] == 'running': break elif r['status'] == 'error': r = self._client.get('clusters/%s/log' % self._cluster_id) print r raise Exception('ERROR creating cluster') if sleeps > 9: raise Exception('Cluster never moved into running state') sleeps += 1 # --------------------------------------------------------------------- def create_omega3p_script(self, omega3p_filename, name=None, number_of_tasks=1): '''Creates script to submit omega3p job ''' command = 'srun -n %s /project/projectdirs/ace3p/{{machine}}/omega3p %s' % \ (number_of_tasks, omega3p_filename) if name is None: name = omega3p_filename body = {'commands': [command], 'name': name} r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] print 'script_id', self._script_id # --------------------------------------------------------------------- def create_input(self, input_paths, folder_name='input_files'): '''DEPRECATED Uploads input files ''' folder_id = self.get_folder(self._private_folder_id, folder_name) if folder_id is None: return print 'input_folder_id', folder_id self._input_folder_id = folder_id def upload_file(path): name = os.path.basename(path) size = os.path.getsize(path) with open(path, 'rb') as fp: self._client.uploadFile(self._input_folder_id, fp, name, size, parentType='folder') for input_path in input_paths: if not input_path or not os.path.exists(input_path): raise Exception('Input file not found: %s' % input_path) upload_file(input_path) # --------------------------------------------------------------------- def create_output_folder(self, folder_name='output_files'): '''DEPRECATED ''' folder_id = self.get_folder(self._private_folder_id, folder_name) print 'output_folder_id', folder_id self._output_folder_id = folder_id # --------------------------------------------------------------------- def create_job(self, job_name, tail=None): ''' ''' # Create job folders folder_name = uuid.uuid4().hex # unique name self._job_folder_id = self.get_folder(self._private_folder_id, folder_name) print 'Created job folder', folder_name self._input_folder_id = self.get_folder(self._job_folder_id, 'input_files') self._output_folder_id = self.get_folder(self._job_folder_id, 'output_files') # Make sure job_name isn't null if not job_name: job_name = 'CumulusJob' # Create job spec body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [{ 'folderId': self._input_folder_id, 'path': '.' }] } if tail: body['output'].append({"path": tail, "tail": True}) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] print 'Created job_id', self._job_id # --------------------------------------------------------------------- def upload_inputs(self, input_paths): '''Uploads input files to input folder ''' if not self._input_folder_id: raise Exception('Input folder missing') def upload_file(path): name = os.path.basename(path) size = os.path.getsize(path) with open(path, 'rb') as fp: self._client.uploadFile(self._input_folder_id, fp, name, size, parentType='folder') for input_path in input_paths: if not input_path or not os.path.exists(input_path): raise Exception('Input file not found: %s' % input_path) upload_file(input_path) # --------------------------------------------------------------------- def submit_job(self, machine, project_account, timeout_minutes, queue='debug', qos=None, number_of_nodes=1, job_output_dir=None): ''' ''' body = { 'machine': machine, 'account': project_account, 'numberOfNodes': number_of_nodes, 'maxWallTime': { 'hours': 0, 'minutes': timeout_minutes, 'seconds': 0 }, 'queue': queue, } if qos: body['qualityOfService'] = qos #print 'jobOutputDir', job_output_dir if job_output_dir: body['jobOutputDir'] = job_output_dir print 'Setting jobOutputDir', job_output_dir url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(body)) print 'Submitted job', self._job_id # --------------------------------------------------------------------- def monitor_job(self, tail=None): '''Periodically monitors job status ''' log_offset = 0 job_timeout = 60 * timeout_minutes start = time.time() while True: time.sleep(2) # Provide some feedback at startup if log_offset == 0: sys.stdout.write('.') #print 'Checking status' r = self._client.get('jobs/%s' % self._job_id) #print r if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) raise Exception(str(r)) elif r['status'] == 'complete': break # Tail log file if tail: params = {'offset': log_offset, 'path': tail} #print 'Checking tail' r = self._client.get('jobs/%s/output' % self._job_id, parameters=params) #print r output = r['content'] if output and log_offset == 0: print # end the user feedback dots log_offset += len(output) for l in output: print l sys.stdout.flush() if time.time() - start > job_timeout: raise Exception('Job timeout') # --------------------------------------------------------------------- def download_results(self, destination_folder): '''Downloads all output files to a local directory ''' if not os.path.exists(destination_folder): os.makedirs(destination_folder) self._client.downloadFolderRecursive(self._output_folder_id, destination_folder) print 'Downloaded files to %s' % destination_folder # --------------------------------------------------------------------- def release_resources(self): '''Closes/deletes any current resources ''' resource_info = { 'clusters': [self._cluster_id], 'jobs': [self._job_id], 'scripts': [self._script_id], 'folder': [self._job_folder] } for resource_type, id_list in resource_info.items(): for resource_id in id_list: if resource_id is not None: url = '%s/%s' % (resource_type, resource_id) self._client.delete(url) self._input_folder_id = None self._job_folder_id = None self._job_id = None self._output_folder_id = None self._script_id = None # --------------------------------------------------------------------- def get_folder(self, parent_id, name): '''Returns folder_id, creating one if needed ''' # Check if folder already exists folder_list = self._client.listFolder(parent_id, name=name) if folder_list: folder = folder_list[0] #print 'found folder %s: %s' % (name, str(folder)) return folder['_id'] # (else) try: r = self._client.createFolder(parent_id, name) return r['_id'] except HttpError as e: print e.responseText return None
def main(args=None): parser = argparse.ArgumentParser( description='Mount Girder filesystem assetstore.') parser.add_argument('--api-url', required=True, default=None, help='full URL to the RESTful API of Girder server') parser.add_argument('--username', required=False, default=None) parser.add_argument('--password', required=False, default=None) parser.add_argument('--api-key', required=False, default=None) parser.add_argument('--token', required=False, default=None) parser.add_argument('--foreground', dest='foreground', action='store_true') parser.add_argument('--hostns', dest='hostns', action='store_true') parser.add_argument('-c', default='remote', help='command to run', choices=['remote', 'direct', 'wt_dms', 'wt_home']) parser.add_argument('local_folder', help='path to local target folder') parser.add_argument('remote_folder', help='Girder\'s folder id or a DM session id') args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get('HOSTDIR', '/'), 'proc/1/ns/mnt') with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == 'remote': FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'direct': FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'wt_dms': FUSE(WtDmsGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True) elif args.c == 'wt_home': user = gc.get('/user/me') args = { 'user': user['login'], 'pass': '******'.format(gc.token), 'dest': args.local_folder, 'opts': '-o uid=1000,gid=100', # FIXME 'url': gc.urlBase.replace('api/v1', 'homes').rstrip('/') # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME else: print('No implementation for command %s' % args.c)
| | / / _/ | / |/ / ____/ | | / /___ _____/ /_____ _____ | | / // // /| | / /|_/ / __/ | | /| / / __ \/ ___/ //_/ _ \/ ___/ | |/ // // ___ |/ / / / /___ | |/ |/ / /_/ / / / ,< / __/ / |___/___/_/ |_/_/ /_/_____/ |__/|__/\____/_/ /_/|_|\___/_/ You are running in private standalone mode. Troubleshooting: Try running `docker pull kitware/viame-worker` to get the latest image Documentation: https://kitware.github.io/dive/Deployment-Docker-Compose/ Issues: https://github.com/Kitware/dive/issues Support: please email [email protected] """) # Fetch Celery broker credentials from server diveclient = GirderClient(apiUrl=dive_api_url) diveclient.authenticate(username=dive_username, password=dive_password) me = diveclient.get('user/me') creds = diveclient.post(f'rabbit_user_queues/user/{me["_id"]}') broker_url = creds['broker_url'] queue_name = f"{me['login']}@private" if not me.get(UserPrivateQueueEnabledMarker, False): warn(" Private queues not enabled for this user.") warn( " You can visit https://viame.kitware/com/#jobs to change these settings" ) info("========================") task_default_queue = queue_name if broker_url is None: raise RuntimeError('CELERY_BROKER_URL must be set') worker_send_task_events = False
except: c.sendRestRequest( "POST", "user", { "login": "******", "password": "******", "email": "*****@*****.**", "firstName": "Girder", "lastName": "Admin", }, ) c.authenticate("girder", "girder") # Create a tangelo hub collection if there isn't one coll_search = c.get("resource/search", parameters={"q": "Default", "types": '["collection"]'}) if len(coll_search["collection"]) == 0: collection = c.post( "collection", parameters={"name": "Default", "description": "Default workspace", "public": "true"} ) c.post( "folder", parameters={ "parentType": "collection", "parentId": collection["_id"], "name": "Data", "description": "Data Folder", "public": "true", }, ) c.post(
def main(args=None): parser = argparse.ArgumentParser(description="Mount Girder filesystem assetstore.") parser.add_argument( "--api-url", required=True, default=None, help="full URL to the RESTful API of Girder server", ) parser.add_argument("--username", required=False, default=None) parser.add_argument("--password", required=False, default=None) parser.add_argument("--api-key", required=False, default=None) parser.add_argument("--token", required=False, default=None) parser.add_argument("--foreground", dest="foreground", action="store_true") parser.add_argument("--hostns", dest="hostns", action="store_true") parser.add_argument( "--versions-mountpoint", dest="versions_mountpoint", required=False, help="Mountpoint for the versions FS. If relative, then it should be " "relative to the runs mountpoint", default="Versions", ) parser.add_argument( "-c", default="remote", help="type of filesystem to mount", choices=[ "remote", "direct", "wt_dms", "wt_home", "wt_work", "wt_run", "wt_versions", "wt_runs", ], ) parser.add_argument("local_folder", help="path to local target folder") parser.add_argument( "remote_folder", help="Girder's folder id, a DM session id (for wt_dms), or a tale instance" "ID (for wt_versions)", ) args = parser.parse_args() gc = GirderClient(apiUrl=args.api_url) if args.token: gc.token = args.token elif args.api_key: gc.authenticate(apiKey=args.api_key) elif args.username and args.password: gc.authenticate(username=args.username, password=args.password) else: raise RuntimeError("You need to specify apiKey or user/pass") if args.hostns: targetns = os.path.join(os.environ.get("HOSTDIR", "/"), "proc/1/ns/mnt") with open(targetns) as fd: setns(fd, CLONE_NEWNS) if args.c == "remote": FUSE( RESTGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "direct": FUSE( LocalGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "wt_dms": FUSE( WtDmsGirderFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=True, allow_other=True, ) elif args.c == "wt_run": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "runId": args.remote_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "runs").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{runId} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_work": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "tale": args.remote_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "tales").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{tale} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_home": user = gc.get("/user/me") args = { "user": user["login"], "pass": "******".format(gc.token), "dest": args.local_folder, "opts": "-o uid=1000,gid=100,file_mode=0600,dir_mode=2700", # FIXME "url": gc.urlBase.replace("api/v1", "homes").rstrip("/"), # FIXME } cmd = 'echo "{user}\n{pass}" | mount.davfs {opts} {url}/{user} {dest}' cmd = cmd.format(**args) subprocess.check_output(cmd, shell=True) # FIXME elif args.c == "wt_versions": FUSE( WtVersionsFS(args.remote_folder, gc), args.local_folder, foreground=args.foreground, ro=False, allow_other=True, ) elif args.c == "wt_runs": FUSE( WtRunsFS(args.remote_folder, gc, args.versions_mountpoint), args.local_folder, foreground=args.foreground, ro=False, allow_other=True, ) else: print("No implementation for command %s" % args.c)
parser = argparse.ArgumentParser() parser.add_argument("path", type=str, help="path to Arbor web apps") parser.add_argument("-g", "--girder-host", type=str, default='localhost', help="host to Girder instance") parser.add_argument("-p", "--girder-port", type=int, default=9000, help="port to Girder instance") args = parser.parse_args() # Get the ID for our Analyses folder. c = GirderClient(host=args.girder_host, port=args.girder_port) c.authenticate('girder', 'girder') folderSearch = c.get('resource/search', parameters={ 'q': 'Analyses', 'types': '["folder"]' }) folderId = folderSearch['folder'][0]['_id'] # Disable authorization requirements for running romanesco tasks c.put('system/setting', parameters={ 'key': 'flow.require_auth', 'value': 'false' }) # Check if these analyses already exist. If so, we won't re-upload them. uploadACR = False uploadPGS = False searchACR = c.get('resource/search', { 'q': 'aceArbor',
class NewtIntegrationTest(BaseIntegrationTest): def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60*5): super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout) self._cluster_id = None self._machine = machine def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post('https://newt.nersc.gov/newt/auth', { 'username': self._girder_user, 'password': self._girder_password}) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()['newt_sessionid'] # Now authenticate with Girder using the session id url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') self._user_id = user['_id'] r = self._client.listFolder(self._user_id, 'user', name='Private') self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): super(NewtIntegrationTest, self).tearDown() if self._cluster_id: try: url = 'clusters/%s' % self._cluster_id self._client.delete(url) except Exception: traceback.print_exc() def create_cluster(self): body = { 'config': { 'host': self._machine }, 'name': 'NewtIntegrationTest', 'type': 'newt' } r = self._client.post('clusters', data=json.dumps(body)) self._cluster_id = r['_id'] # Now test the connection r = self._client.put('clusters/%s/start' % self._cluster_id) sleeps = 0 while True: time.sleep(1) r = self._client.get('clusters/%s/status' % self._cluster_id) if r['status'] == 'running': break elif r['status'] == 'error': r = self._client.get('clusters/%s/log' % self._cluster_id) self.fail(str(r)) if sleeps > 9: self.fail('Cluster never moved into running state') sleeps += 1 def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) url = '%s/api/v1/file/%s/download' % (self._girder_url, r[0]['_id']) r = self._session.get(url) self.assertEqual(r.content, self._data) def test(self): try: self.create_cluster() self.create_script() self.create_input() self.create_output_folder() self.create_job() self.submit_job(timeout=self._job_timeout) self.assert_output() except HttpError as error: self.fail(error.responseText)
class NewtIntegrationTest(BaseIntegrationTest): def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60 * 5): super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout) self._cluster_id = None self._machine = machine def setUp(self): # First authenticate with NEWT self._session = Session() r = self._session.post('https://newt.nersc.gov/newt/auth', { 'username': self._girder_user, 'password': self._girder_password }) self.assertEqual(r.status_code, 200) print r.json() self._newt_session_id = r.json()['newt_sessionid'] # Now authenticate with Girder using the session id url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id) r = self._session.put(url) self.assertEqual(r.status_code, 200) url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.token = self._session.cookies['girderToken'] user = self._client.get('user/me') self._user_id = user['_id'] r = self._client.listFolder(self._user_id, 'user', name='Private') r = list(r) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): super(NewtIntegrationTest, self).tearDown() if self._cluster_id: try: url = 'clusters/%s' % self._cluster_id self._client.delete(url) except Exception: traceback.print_exc() def create_cluster(self): body = { 'config': { 'host': self._machine }, 'name': 'NewtIntegrationTest', 'type': 'newt' } r = self._client.post('clusters', data=json.dumps(body)) self._cluster_id = r['_id'] # Now test the connection r = self._client.put('clusters/%s/start' % self._cluster_id) sleeps = 0 while True: time.sleep(1) r = self._client.get('clusters/%s/status' % self._cluster_id) if r['status'] == 'running': break elif r['status'] == 'error': r = self._client.get('clusters/%s/log' % self._cluster_id) self.fail(str(r)) if sleeps > 9: self.fail('Cluster never moved into running state') sleeps += 1 def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) url = '%s/api/v1/file/%s/download' % (self._girder_url, r[0]['_id']) r = self._session.get(url) self.assertEqual(r.content, self._data) def test(self): try: self.create_cluster() self.create_script() self.create_input() self.create_output_folder() self.create_job() self.submit_job(timeout=self._job_timeout) self.assert_output() except HttpError as error: self.fail(error.responseText)
c.authenticate('girder', 'girder') except: c.sendRestRequest( 'POST', 'user', { "login": "******", "password": "******", "email": "*****@*****.**", "firstName": "Girder", "lastName": "Admin" }) c.authenticate('girder', 'girder') # Create a tangelo hub collection if there isn't one coll_search = c.get('resource/search', parameters={ 'q': 'Default', 'types': '["collection"]' }) if len(coll_search["collection"]) == 0: collection = c.post('collection', parameters={ 'name': 'Default', 'description': 'Default workspace', 'public': 'true' }) c.post('folder', parameters={ 'parentType': 'collection', 'parentId': collection['_id'], 'name': 'Data', 'description': 'Data Folder',
class BaseIntegrationTest(unittest.TestCase): def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True): super(BaseIntegrationTest, self).__init__(name) self._job_id = None self._script_id = None self._output_folder_id = None self._input_folder_id = None self._girder_url = girder_url self._girder_user = girder_user self._girder_password = girder_password self._job_timeout = job_timeout self._data = 'Need more input!' self._cleanup = cleanup def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): if not self._cleanup: return if self._job_id: try: url = 'jobs/%s' % self._job_id self._client.delete(url) except Exception as e: traceback.print_exc() if self._script_id: try: url = 'scripts/%s' % self._script_id self._client.delete(url) except Exception: traceback.print_exc() if self._output_folder_id: try: url = 'folder/%s' % self._output_folder_id self._client.delete(url) except Exception: traceback.print_exc() if self._input_folder_id: try: url = 'folder/%s' % self._input_folder_id self._client.delete(url) except Exception: traceback.print_exc() def create_script(self, commands=[ 'sleep 10', 'cat CumulusIntegrationTestInput' ]): body = { 'commands': commands, 'name': 'CumulusIntegrationTestLob' } r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] def create_input(self, folder_name='CumulusInput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._input_folder_id = r['_id'] size = len(self._data) item = self._client.uploadFile(self._input_folder_id, StringIO(self._data), 'CumulusIntegrationTestInput', size, parentType='folder') self._item_id = item['itemId'] def create_output_folder(self, folder_name='CumulusOutput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._output_folder_id = r['_id'] def create_job(self, job_name='CumulusIntegrationTestJob', tail=None): body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [ { 'folderId': self._input_folder_id, 'path': '.' } ] } if tail: body['output'].append({ "path": tail, "tail": True }) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] def submit_job(self, job_params={}, timeout=None): url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(job_params)) start = time.time() while True: time.sleep(1) r = self._client.get('jobs/%s' % self._job_id) if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) self.fail(str(r)) elif r['status'] == 'complete': break if time.time() - start > timeout: self.fail('Job didn\'t complete in timeout') def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) path = os.path.join(tempfile.gettempdir(), self._job_id) try: self._client.downloadFile(r[0]['_id'], path) with open(path, 'rb') as fp: self.assertEqual(fp.read(), self._data) finally: if os.path.exists(path): os.remove(path)
for root, dirs, files in os.walk(arbor_collections_path): for file in files: if not file.endswith(".json"): continue # Get the name of this file and the directory that it's in. # We use the directory name as the collection name in Girder. fullpath = os.path.join(root, file) analysis_filename = os.path.basename(fullpath) analysis_name = os.path.splitext(analysis_filename)[0] analysis_dir = os.path.dirname(fullpath) collection_name = os.path.basename(analysis_dir) # Create this collection if it doesn't already exist. collection_search = c.get('resource/search', parameters={ 'q': collection_name, 'types': '["collection"]' }) if len(collection_search["collection"]) == 0: collection = c.post('collection', parameters={ 'name': collection_name, 'description': collection_name, 'public': 'true' }) c.post('folder', parameters={ 'parentType': 'collection', 'parentId': collection['_id'], 'name': 'Data', 'description': 'Data Folder', 'public': 'true' }) c.post('folder', parameters={
'gridWidth': lon_select_index, 'gridHeight': lat_select_index, 'x0': float(data.variables[lon_name][0]), 'y0': float(data.variables[lat_name][0]), 'dx': float(data.variables[lon_name][1] - data.variables[lon_name][0]), 'dy': float(data.variables[lat_name][1] - data.variables[lat_name][0]), 'values': variable[timestep][:lat_select_index, :lon_select_index].reshape(variable[timestep][:lat_select_index, :lon_select_index].size).tolist() } return contour_data client = GirderClient(host, port) client.token = token # Get the user user = client.get('user/me') # Get the dataset folder parameters = { 'userId': user['_id'] } dataset_folder = client.get('minerva_dataset/folder', parameters=parameters)['folder'] dataset_folder_id = dataset_folder['_id'] parameters = { 'id': fileId, 'type': 'file' } # Get the file resource so we can get the name input_file = client.get('resource/%s' % str(fileId), parameters=parameters) input_file_name = input_file['name'] output_file_name = input_file_name.replace('.nc', '.json')
from girder_client import GirderClient import json import pymongo import sys if len(sys.argv) < 2: print "%s /path/to/ArborWebApps" % sys.argv[0] sys.exit(1) arborWebAppsPath = sys.argv[1] # Get the ID for our Analyses folder. c = GirderClient(host='localhost', port=9000) c.authenticate('girder', 'girder') folderSearch = c.get('resource/search', parameters={ 'q': 'Analyses', 'types': '["folder"]' }) folderId = folderSearch['folder'][0]['_id'] # Disable authorization requirements for running romanesco tasks c.put('system/setting', parameters={ 'key': 'romanesco.require_auth', 'value': 'false' }) # Check if these analyses already exist. If so, we won't re-upload them. uploadACR = False uploadPGS = False
class BaseIntegrationTest(unittest.TestCase): def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True): super(BaseIntegrationTest, self).__init__(name) self._job_id = None self._script_id = None self._output_folder_id = None self._input_folder_id = None self._girder_url = girder_url self._girder_user = girder_user self._girder_password = girder_password self._job_timeout = job_timeout self._data = 'Need more input!' self._cleanup = cleanup def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): if not self._cleanup: return if self._job_id: try: url = 'jobs/%s' % self._job_id self._client.delete(url) except Exception as e: traceback.print_exc() if self._script_id: try: url = 'scripts/%s' % self._script_id self._client.delete(url) except Exception: traceback.print_exc() if self._output_folder_id: try: url = 'folder/%s' % self._output_folder_id self._client.delete(url) except Exception: traceback.print_exc() if self._input_folder_id: try: url = 'folder/%s' % self._input_folder_id self._client.delete(url) except Exception: traceback.print_exc() def create_script(self, commands=['sleep 10', 'cat CumulusIntegrationTestInput']): body = {'commands': commands, 'name': 'CumulusIntegrationTestLob'} r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] def create_input(self, folder_name='CumulusInput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._input_folder_id = r['_id'] size = len(self._data) item = self._client.uploadFile(self._input_folder_id, StringIO(self._data), 'CumulusIntegrationTestInput', size, parentType='folder') self._item_id = item['itemId'] def create_output_folder(self, folder_name='CumulusOutput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._output_folder_id = r['_id'] def create_job(self, job_name='CumulusIntegrationTestJob', tail=None): body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [{ 'folderId': self._input_folder_id, 'path': '.' }] } if tail: body['output'].append({"path": tail, "tail": True}) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] def submit_job(self, job_params={}, timeout=None): url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(job_params)) start = time.time() while True: time.sleep(1) r = self._client.get('jobs/%s' % self._job_id) if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) self.fail(str(r)) elif r['status'] == 'complete': break if time.time() - start > timeout: self.fail('Job didn\'t complete in timeout') def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) path = os.path.join(tempfile.gettempdir(), self._job_id) try: self._client.downloadFile(r[0]['_id'], path) with open(path, 'rb') as fp: self.assertEqual(fp.read(), self._data) finally: if os.path.exists(path): os.remove(path)