def testGetApiUrl(self): url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test' self.assertEqual(rest.getApiUrl(url), 'https://localhost/thing/api/v1') parts = rest.getUrlParts(url) self.assertEqual(parts.path, '/thing/api/v1/hello/world') self.assertEqual(rest.getApiUrl(parts.path), '/thing/api/v1') self.assertEqual(parts.port, None) self.assertEqual(parts.hostname, 'localhost') self.assertEqual(parts.query, 'foo=bar') self.assertEqual(parts.fragment, 'test') url = 'https://localhost/girder#users' self.assertRaises(Exception, rest.getApiUrl, url=url)
def testGetApiUrl(): url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test' assert rest.getApiUrl(url) == 'https://localhost/thing/api/v1' parts = rest.getUrlParts(url) assert parts.path == '/thing/api/v1/hello/world' assert rest.getApiUrl(parts.path) == '/thing/api/v1' assert parts.port is None assert parts.hostname == 'localhost' assert parts.query == 'foo=bar' assert parts.fragment == 'test' url = 'https://localhost/girder#users' with pytest.raises(GirderException, match='Could not determine API root in %s.$' % url): rest.getApiUrl(url)
def nearestNeighborIndex(item, user, descriptorSet): """ Get the nearest neighbor index from a given item and descriptor set. :param item: Item to find the nn index from, usually the item that the user is performing the nearest neighbors search on. :param user: The owner of the .smqtk folder. :param descriptorSet: The relevant descriptor set. """ folder = ModelImporter.model('folder') _GirderDataElement = functools.partial(GirderDataElement, api_root=getApiUrl(), token=getCurrentToken()['_id']) smqtkFolder = folder.createFolder(folder.load(item['folderId'], user=user), '.smqtk', reuseExisting=True) try: meanVecFileId = localSmqtkFileIdFromName(smqtkFolder, 'mean_vec.npy') rotationFileId = localSmqtkFileIdFromName(smqtkFolder, 'rotation.npy') hash2uuidsFileId = localSmqtkFileIdFromName(smqtkFolder, 'hash2uuids.pickle') except Exception: logger.warn('SMQTK files didn\'t exist for performing NN on %s' % item['_id']) return None # TODO Should these be Girder data elements? Unnecessary HTTP requests. functor = ItqFunctor(mean_vec_cache=_GirderDataElement(meanVecFileId), rotation_cache=_GirderDataElement(rotationFileId)) hash2uuidsKV = MemoryKeyValueStore(_GirderDataElement(hash2uuidsFileId)) return LSHNearestNeighborIndex(functor, descriptorSet, hash2uuidsKV, read_only=True)
def getUrl(cls, state): clientId = Setting().get(PluginSettings.SYNAPSE_CLIENT_ID) if not clientId: raise Exception('No Synapse client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'synapse', 'callback')) # OIDC claims. The complete list is here: # https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/oauth/OIDCClaimName.html claims = { 'email': None, 'given_name': None, 'family_name': None } tokenAndUserInfoClaims = {'id_token': claims, 'userinfo': claims} query = urllib.parse.urlencode({ 'response_type': 'code', 'access_type': 'online', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': 'openid %s' % ' '.join(cls._AUTH_SCOPES), 'claims': json.dumps(tokenAndUserInfoClaims) # Synapse supports a 'nonce', but does not require it, and Girder's particular checks # and invalidation of the 'state' token provides the same security guarantees }) return '%s?%s' % (cls._AUTH_URL, query)
def girderOutputSpec(parent, token, parentType='folder', name=None, dataType='string', dataFormat='text'): """ Downstream plugins that are building worker jobs that use Girder IO should use this to generate the output specs more easily. :param parent: The parent to upload the data into (an item or folder). :type parent: dict :param token: The Girder token document or raw token string to use to authenticate when uploading. :type token: dict or str :param parentType: The type of the parent object ("item" or "folder"). :type parentType: str :param name: Name of the resource to use when uploading. Required if the output target type is "memory". If the target is "filepath", uses the basename of the file being uploaded by default. :type name: str or None :param dataType: The worker `type` field. :type dataType: str :param dataFormat: The worker `format` field. :type dataFormat: str """ if isinstance(token, dict): token = token['_id'] return { 'mode': 'girder', 'api_url': getApiUrl(), 'token': token, 'name': name, 'parent_id': str(parent['_id']), 'parent_type': parentType, 'type': dataType, 'format': dataFormat }
def girderInputSpec(resource, resourceType='file', name=None, token=None, dataType='string', dataFormat='text'): """ Downstream plugins that are building Girder worker jobs that use Girder IO should use this to generate the input specs more easily. :param resource: The resource document to be downloaded at runtime. :type resource: dict :param resourceType: The resource type to download for the input. Should be "folder", "item", or "file". :type resourceType: str :param name: The name of the resource to download. If not passed, uses the "name" field of the resource document. :type name: str or None :param token: The Girder token document or raw token string to use to authenticate when downloading. Pass `None` for anonymous downloads. :type token: dict, str, or None :param dataType: The worker `type` field. :type dataType: str :param dataFormat: The worker `format` field. :type dataFormat: str """ if isinstance(token, dict): token = token['_id'] return { 'mode': 'girder', 'api_url': getApiUrl(), 'token': token, 'id': str(resource['_id']), 'name': name or resource['name'], 'resource_type': resourceType, 'type': dataType, 'format': dataFormat }
def testStream(self, item, params): token = self.getCurrentToken() jobModel = self.model('job', 'jobs') job = jobModel.createJob(title='docker stream test', type='docker_test', handler='worker_handler', user=self.getCurrentUser()) jobToken = jobModel.createJobToken(job) apiUrl = getApiUrl() kwargs = { 'task': { 'mode': 'docker', 'docker_image': 'testoutputs:latest', 'pull_image': False, 'inputs': [{ 'id': 'input_pipe', 'target': 'filepath', 'stream': True }], 'outputs': [{ 'id': 'output_pipe', 'target': 'filepath', 'stream': True }] }, 'inputs': { 'input_pipe': { 'mode': 'http', 'method': 'GET', 'url': '%s/item/%s/download' % (apiUrl, str(item['_id'])), 'headers': { 'Girder-Token': str(token['_id']) } } }, 'outputs': { 'output_pipe': { 'mode': 'http', 'method': 'POST', 'url': apiUrl + '/docker_test/stream_callback', 'headers': { 'Girder-Token': str(token['_id']) } } }, 'validate': False, 'auto_convert': False, 'cleanup': False, 'jobInfo': utils.jobInfoSpec(job, jobToken) } job['kwargs'] = kwargs job = jobModel.save(job) jobModel.scheduleJob(job) return job
def jobInfoSpec(job, token=None, logPrint=True): """ Build the jobInfo specification for a task to write status and log output back to a Girder job. :param job: The job document representing the worker task. :type job: dict :param token: The token to use. Creates a job token if not passed. :type token: str or dict :param logPrint: Whether standard output from the job should be """ if token is None: token = ModelImporter.model('job', 'jobs').createJobToken(job) if isinstance(token, dict): token = token['_id'] return { 'method': 'PUT', 'url': '/'.join((getApiUrl(), 'job', str(job['_id']))), 'reference': str(job['_id']), 'headers': { 'Girder-Token': token }, 'logPrint': logPrint }
def testServerRootSetting(self): settingModel = Setting() with self.assertRaises(ValidationException): settingModel.set(SettingKey.SERVER_ROOT, 'bad_value') settingModel.set(SettingKey.SERVER_ROOT, 'https://somedomain.org/foo') self.assertEqual(getApiUrl(), 'https://somedomain.org/foo/api/v1')
def testServerRootSetting(self): settingModel = self.model('setting') with self.assertRaises(ValidationException): settingModel.set(SettingKey.SERVER_ROOT, 'bad_value') settingModel.set(SettingKey.SERVER_ROOT, 'https://somedomain.org/foo') self.assertEqual(getApiUrl(), 'https://somedomain.org/foo/api/v1')
def nearestNeighborIndex(item, user, descriptorIndex): """ Get the nearest neighbor index from a given item and descriptor index. :param item: Item to find the nn index from, usually the item that the user is performing the nearest neighbors search on. :param user: The owner of the .smqtk folder. :param descriptorIndex: The relevant descriptor index. """ folder = ModelImporter.model('folder') _GirderDataElement = functools.partial(GirderDataElement, api_root=getApiUrl(), token=getCurrentToken()['_id']) smqtkFolder = folder.createFolder(folder.load(item['folderId'], user=user), '.smqtk', reuseExisting=True) try: meanVecFileId = localSmqtkFileIdFromName(smqtkFolder, 'mean_vec.npy') rotationFileId = localSmqtkFileIdFromName(smqtkFolder, 'rotation.npy') hash2uuidsFileId = localSmqtkFileIdFromName(smqtkFolder, 'hash2uuids.pickle') except Exception: logger.warn('SMQTK files didn\'t exist for performing NN on %s' % item['_id']) return None # TODO Should these be Girder data elements? Unnecessary HTTP requests. functor = ItqFunctor(mean_vec_cache=_GirderDataElement(meanVecFileId), rotation_cache=_GirderDataElement(rotationFileId)) hash2uuidsKV = MemoryKeyValueStore(_GirderDataElement(hash2uuidsFileId)) return LSHNearestNeighborIndex(functor, descriptorIndex, hash2uuidsKV, read_only=True)
def provision(self): self.status = ClusterStatus.PROVISIONING base_url = getApiUrl() log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id']) girder_token = get_task_token()['_id'] profile, secret_key = _get_profile(self.cluster['profileId']) playbook = get_property( 'config.provision.spec', self.cluster, default=self.DEFAULT_PLAYBOOK) playbook_params = get_property( 'config.provision.params', self.cluster, default={}) provision_ssh_user = get_property( 'config.provision.ssh.user', self.cluster, default='ubuntu') playbook_params['cluster_state'] = ClusterStatus.RUNNING playbook_params['ansible_ssh_user'] = provision_ssh_user cumulus.ansible.tasks.cluster.provision_cluster \ .delay(playbook, self._model.filter(self.cluster, getCurrentUser(), passphrase=False), profile, secret_key, playbook_params, girder_token, log_write_url, ClusterStatus.RUNNING) return self.cluster
def submit_job(self, job): log_url = '%s/jobs/%s/log' % (getApiUrl(), job['_id']) girder_token = get_task_token(self.cluster)['_id'] cumulus.tasks.job.submit( girder_token, self._model.filter(self.cluster, getCurrentUser(), passphrase=False), job, log_url)
def getWorkerApiUrl(): """ Return the API base URL to which the worker should callback to write output information back to the server. This is controlled via a system setting, and the default is to use the core server root setting. """ apiUrl = ModelImporter.model('setting').get(PluginSettings.API_URL) return apiUrl or getApiUrl()
def getWorkerApiUrl(): """ Return the API base URL to which the worker should callback to write output information back to the server. This is controlled via a system setting, and the default is to use the core server root setting. """ apiUrl = Setting().get(PluginSettings.API_URL) return apiUrl or getApiUrl()
def start(self, request_body): log_write_url = '%s/clusters/%s/log' % (getApiUrl(), self.cluster['_id']) girder_token = get_task_token(self.cluster)['_id'] cumulus.tasks.cluster.test_connection \ .delay( self._model.filter(self.cluster, getCurrentUser(), passphrase=False), log_write_url=log_write_url, girder_token=girder_token)
def createBlurImage(self, item, params): user = self.getCurrentUser() token = self.getCurrentToken() jobTitle = 'ITK blur: ' + item['name'] jobModel = self.model('job', 'jobs') folder = self.model('folder').load(item['folderId'], force=True) job = jobModel.createJob( title=jobTitle, type='itk_blur', handler='worker_handler', user=user) jobToken = jobModel.createJobToken(job) scriptFile = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts', 'cad_script.py') with open(scriptFile, 'r') as fh: script = fh.read() kwargs = { 'task': { 'name': jobTitle, 'mode': 'python', 'script': script, 'inputs': [{ 'id': 'inputFileName', 'type': 'string', 'format': 'text', 'target': 'filepath' }], 'outputs': [{ 'id': 'outputFileName', 'format': 'text', 'type': 'string', 'target': 'filepath' }] }, 'inputs': { 'inputFileName': utils.girderInputSpec( item, resourceType='item', token=token) }, 'outputs': { 'outputFileName': utils.girderOutputSpec( folder, token=token, parentType='folder') }, 'jobInfo': { 'method': 'PUT', 'url': '/'.join((getApiUrl(), 'job', str(job['_id']))), 'headers': {'Girder-Token': jobToken['_id']}, 'logPrint': True }, 'validate': False, 'auto_convert': False } job['kwargs'] = kwargs job = jobModel.save(job) jobModel.scheduleJob(job) return job
def testStream(self, item, params): token = self.getCurrentToken() jobModel = self.model('job', 'jobs') job = jobModel.createJob( title='docker stream test', type='docker_test', handler='worker_handler', user=self.getCurrentUser()) jobToken = jobModel.createJobToken(job) apiUrl = getApiUrl() kwargs = { 'task': { 'mode': 'docker', 'docker_image': 'testoutputs:latest', 'pull_image': False, 'inputs': [{ 'id': 'input_pipe', 'target': 'filepath', 'stream': True }], 'outputs': [{ 'id': 'output_pipe', 'target': 'filepath', 'stream': True }] }, 'inputs': { 'input_pipe': { 'mode': 'http', 'method': 'GET', 'url': '%s/item/%s/download' % (apiUrl, str(item['_id'])), 'headers': { 'Girder-Token': str(token['_id']) } } }, 'outputs': { 'output_pipe': { 'mode': 'http', 'method': 'POST', 'url': apiUrl + '/docker_test/stream_callback', 'headers': { 'Girder-Token': str(token['_id']) } } }, 'validate': False, 'auto_convert': False, 'cleanup': False, 'jobInfo': utils.jobInfoSpec(job, jobToken) } job['kwargs'] = kwargs job = jobModel.save(job) jobModel.scheduleJob(job) return job
def _startContainer(self, container): settings = Setting() psRoot = settings.get(PluginSettings.PRIVATE_STORAGE_PATH) restUrl = rest.getApiUrl() token = rest.getCurrentToken()['_id'] sessionId = str(container['sessionId']) mountId = efs.mount(sessionId, '/tmp/' + sessionId, psRoot, restUrl, token) container['mountId'] = mountId container['status'] = 'Running' self.save(container)
def getUrl(cls, state): _, _, redirect = state.partition(".") url = "/".join((getApiUrl(), "account", cls.getProviderName(external=False), "callback")) url += "?state={}&code=dataone".format(quote(state)) auth_url = urlparse(cls.get_cn())._replace( path="/portal/oauth", query="action=start&target={}".format(quote(url))) return urlunparse(auth_url)
def runJsonTasksDescription(self, folder, image, pullImage, params): jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser()) job = jobModel.createJob(title='Read docker task specs: %s' % image, type='item_task.json_description', handler='worker_handler', user=self.getCurrentUser()) jobOptions = { 'itemTaskId': folder['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': [], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'POST', 'format': 'text', 'url': '/'.join((getApiUrl(), self.resourceName, str(folder['_id']), 'json_specs')), 'headers': { 'Girder-Token': token['_id'] }, 'params': { 'image': image, 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False, 'cleanup': True } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def createInstance(self, tale, user, token, name=None, save=True): existing = self.findOne({ 'taleId': tale['_id'], 'creatorId': user['_id'], }) if existing: return existing if not name: name = tale.get('title', '') workspaceFolder = self.model('tale', 'wholetale').createWorkspace(tale) now = datetime.datetime.utcnow() payload = { 'girder_token': str(token['_id']), 'apiUrl': getWorkerApiUrl(), 'taleId': str(tale['_id']), 'workspaceId': str(workspaceFolder['_id']), 'api_version': API_VERSION } volumeTask = getCeleryApp().send_task('gwvolman.tasks.create_volume', args=[payload]) volume = volumeTask.get(timeout=TASK_TIMEOUT) payload.update(volume) serviceTask = getCeleryApp().send_task( 'gwvolman.tasks.launch_container', args=[payload], queue='manager') service = serviceTask.get(timeout=TASK_TIMEOUT) service.update(volume) netloc = urllib.parse.urlsplit(getApiUrl()).netloc domain = '{}.{}'.format(service['name'], netloc.split(':')[0].split('.', 1)[1]) url = 'https://{}/{}'.format(domain, service.get('urlPath', '')) _wait_for_server(url) instance = { 'taleId': tale['_id'], 'created': now, 'creatorId': user['_id'], 'lastActivity': now, 'containerInfo': service, 'name': name, 'status': InstanceStatus.RUNNING, # be optimistic for now 'url': url } self.setUserAccess(instance, user=user, level=AccessType.ADMIN) if save: instance = self.save(instance) return instance
def start(self, request_body): if self.cluster['status'] == ClusterStatus.CREATING: raise RestException('Cluster is not ready to start.', code=400) log_write_url = '%s/clusters/%s/log' % (getApiUrl(), self.cluster['_id']) girder_token = get_task_token()['_id'] cumulus.tasks.cluster.test_connection \ .delay(self._model.filter(self.cluster, getCurrentUser(), passphrase=False), log_write_url=log_write_url, girder_token=girder_token)
def getUrl(cls, state): clientId = Setting().get(PluginSettings.GITHUB_CLIENT_ID) if not clientId: raise Exception('No GitHub client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'github', 'callback')) query = urllib.parse.urlencode({ 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ','.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state): clientId = Setting().get(PluginSettings.BOX_CLIENT_ID) if not clientId: raise Exception('No Box client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'box', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, }) return '%s?%s' % (cls._AUTH_URL, query)
def process(self, workingSet, options, params): """ Run the complete processing workflow. """ user = self.getCurrentUser() apiUrl = getApiUrl() token = getCurrentToken() outputFolder = self._outputFolder(workingSet) requestInfo = RequestInfo(user=user, apiUrl=apiUrl, token=token) workflowManager = DanesfieldWorkflowManager.instance() jobId = workflowManager.initJob(requestInfo, workingSet, outputFolder, options) workflowManager.advance(jobId=jobId)
def getUrl(cls, state): clientId = Setting().get(constants.PluginSettings.GITHUB_CLIENT_ID) if clientId is None: raise Exception('No GitHub client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'github', 'callback')) query = urllib.parse.urlencode({ 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ','.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def _testSettings(self, providerInfo): Setting().set(SettingKey.REGISTRATION_POLICY, 'closed') self.accountType = 'new' # We should get an empty listing when no providers are set up params = {'key': PluginSettings.PROVIDERS_ENABLED, 'value': []} resp = self.request('/system/setting', user=self.adminUser, method='PUT', params=params) self.assertStatusOk(resp) resp = self.request('/nciLogin/loginCallback', exception=True) self.assertStatusOk(resp) self.assertFalse(resp.json) # Set up provider normally params = { 'list': json.dumps([{ 'key': PluginSettings.PROVIDERS_ENABLED, 'value': [providerInfo['id']] }, { 'key': PluginSettings.NCI_CLIENT_ID, 'value': providerInfo['id'] }, { 'key': PluginSettings.NCI_RETURN_URL, 'value': providerInfo['return_url'] }, { 'key': PluginSettings.NCI_LOGIN_URL, 'value': providerInfo['login_url'] }, { 'key': PluginSettings.NCI_VALIDATION_URL, 'value': providerInfo['validation_url'] }]) } resp = self.request('/system/setting', user=self.adminUser, method='PUT', params=params) self.assertStatusOk(resp) # # No need to re-fetch and test all of these settings values; they will # # be implicitly tested later resp = self.request('/nciLogin/loginCallback', exception=True) self.assertStatusOk(resp) expect = 'https://bar.com?returnUrl=' + '/'.join( (getApiUrl(), 'nciLogin', 'callback')) self.assertEqual(resp.json, expect)
def getUrl(cls, state): clientId = Setting().get(constants.PluginSettings.BITBUCKET_CLIENT_ID) if clientId is None: raise Exception('No Bitbucket client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'bitbucket', 'callback')) query = urllib.parse.urlencode({ 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'response_type': 'code', 'scope': ','.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state): clientId = Setting().get(constants.PluginSettings.LINKEDIN_CLIENT_ID) if clientId is None: raise Exception('No LinkedIn client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'linkedin', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ' '.join(cls._AUTH_SCOPES) }) return '?'.join((cls._AUTH_URL, query))
def getUrl(cls, state): clientId = Setting().get(constants.PluginSettings.GLOBUS_CLIENT_ID) if clientId is None: raise Exception('No Globus client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'globus', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'access_type': 'online', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ' '.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state): clientId = cls.model('setting').get( constants.PluginSettings.BITBUCKET_CLIENT_ID) if clientId is None: raise Exception('No Bitbucket client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'bitbucket', 'callback')) query = urllib.parse.urlencode({ 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'response_type': 'code', 'scope': ','.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state): clientId = Setting().get(PluginSettings.GLOBUS_CLIENT_ID) if not clientId: raise Exception('No Globus client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'globus', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'access_type': 'online', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ' '.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def _emitHook(event, hook): body = json.dumps( { 'apiUrl': getApiUrl(), 'name': event.name, 'info': event.info, 'uid': uuid.uuid4() }, cls=JsonEncoder) headers = {'Content-Type': 'application/json'} if 'hmacKey' in hook: headers['Girder-Signature'] = 'sha256=' + hmac.new( hook['hmacKey'].encode('utf8'), body, hashlib.sha256).hexdigest() # TODO this is blocking (because requests is blocking). We should do this asynchronously. requests.post(hook['url'], data=body, headers=headers)
def girderOutputSpec(parent, token, parentType='folder', name=None, dataType='string', dataFormat='text', reference=None): """ Downstream plugins that are building worker jobs that use Girder IO should use this to generate the output specs more easily. :param parent: The parent to upload the data into (an item or folder). :type parent: dict :param token: The Girder token document or raw token string to use to authenticate when uploading. :type token: dict or str :param parentType: The type of the parent object ("item" or "folder"). :type parentType: str :param name: Name of the resource to use when uploading. Required if the output target type is "memory". If the target is "filepath", uses the basename of the file being uploaded by default. :type name: str or None :param dataType: The worker `type` field. :type dataType: str :param dataFormat: The worker `format` field. :type dataFormat: str :param reference: Optional "reference" string to pass back to the server during the upload. This can be used to attach arbitrary data to this for tracking purposes, e.g., referring back to related inputs. :type reference: str """ if isinstance(token, dict): token = token['_id'] return { 'mode': 'girder', 'api_url': getApiUrl(), 'token': token, 'name': name, 'parent_id': str(parent['_id']), 'parent_type': parentType, 'type': dataType, 'format': dataFormat, 'reference': reference }
def getUrl(cls, state): clientId = cls.model('setting').get( constants.PluginSettings.GOOGLE_CLIENT_ID) if clientId is None: raise Exception('No Google client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'google', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'access_type': 'online', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': ' '.join(cls._AUTH_SCOPES) }) return '%s?%s' % (cls._AUTH_URL, query)
def detach(self, volume, params): profile_id = parse('profileId').find(volume)[0].value profile, secret_key = _get_profile(profile_id) girder_callback_info = { 'girder_api_url': getApiUrl(), 'girder_token': get_task_token()['_id']} p = CloudProvider(dict(secretAccessKey=secret_key, **profile)) aws_volume = p.get_volume(volume) if aws_volume is None or aws_volume['state'] != VolumeState.INUSE: raise RestException('This volume is not attached ' 'to a cluster', 400) if 'clusterId' not in volume: raise RestException('clusterId is not set on this volume!', 400) try: volume['path'] except KeyError: raise RestException('path is not set on this volume!', 400) cluster = self.model('cluster', 'cumulus').load(volume['clusterId'], user=getCurrentUser(), level=AccessType.ADMIN) master = p.get_master_instance(cluster['_id']) if master['state'] != InstanceState.RUNNING: raise RestException('Master instance is not running!', 400) user = getCurrentUser() cluster = self.model('cluster', 'cumulus').filter( cluster, user, passphrase=False) cumulus.ansible.tasks.volume.detach_volume\ .delay(profile, cluster, master, self._model.filter(volume, user), secret_key, girder_callback_info) volume['status'] = VolumeState.DETACHING volume = self._model.update_volume(user, volume) return self._model.filter(volume, user)
def girderInputSpec(resource, resourceType='file', name=None, token=None, dataType='string', dataFormat='text', fetchParent=False): """ Downstream plugins that are building Girder worker jobs that use Girder IO should use this to generate the input specs more easily. :param resource: The resource document to be downloaded at runtime. :type resource: dict :param resourceType: The resource type to download for the input. Should be "folder", "item", or "file". :type resourceType: str :param name: The name of the resource to download. If not passed, uses the "name" field of the resource document. :type name: str or None :param token: The Girder token document or raw token string to use to authenticate when downloading. Pass `None` for anonymous downloads. :type token: dict, str, or None :param dataType: The worker `type` field. :type dataType: str :param dataFormat: The worker `format` field. :type dataFormat: str :param fetchParent: Whether to fetch the whole parent resource of the specified resource as a side effect. :type fetchParent: bool """ if isinstance(token, dict): token = token['_id'] return { 'mode': 'girder', 'api_url': getApiUrl(), 'token': token, 'id': str(resource['_id']), 'name': name or resource['name'], 'resource_type': resourceType, 'type': dataType, 'format': dataFormat, 'fetch_parent': fetchParent }
def attach(self, volume, cluster, params): body = getBodyJson() self.requireParams(['path'], body) path = body['path'] profile_id = parse('profileId').find(volume)[0].value profile, secret_key = _get_profile(profile_id) girder_callback_info = { 'girder_api_url': getApiUrl(), 'girder_token': get_task_token()['_id']} p = CloudProvider(dict(secretAccessKey=secret_key, **profile)) aws_volume = p.get_volume(volume) # If volume exists it needs to be available to be attached. If # it doesn't exist it will be created as part of the attach # playbook. if aws_volume is not None and \ aws_volume['state'] != VolumeState.AVAILABLE: raise RestException('This volume is not available to attach ' 'to a cluster', 400) master = p.get_master_instance(cluster['_id']) if master['state'] != InstanceState.RUNNING: raise RestException('Master instance is not running!', 400) cluster = self.model('cluster', 'cumulus').filter( cluster, getCurrentUser(), passphrase=False) cumulus.ansible.tasks.volume.attach_volume\ .delay(profile, cluster, master, self._model.filter(volume, getCurrentUser()), path, secret_key, girder_callback_info) volume['status'] = VolumeState.ATTACHING volume = self._model.update_volume(getCurrentUser(), volume) return self._model.filter(volume, getCurrentUser())
def jobInfoSpec(job, token, logPrint=True): """ Build the jobInfo specification for a romanesco task to write status and log output back to a Girder job. :param job: The job document representing the romanesco task. :type job: dict :param token: The token :type token: str or dict :param logPrint: Whether standard output from the job should be """ if type(token) is dict: token = token['_id'] return { 'method': 'PUT', 'url': '/'.join((getApiUrl(), 'job', str(job['_id']))), 'headers': {'Girder-Token': token}, 'logPrint': logPrint }
def terminate(self): self.status = ClusterStatus.TERMINATING base_url = getApiUrl() log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id']) girder_token = get_task_token()['_id'] profile, secret_key = _get_profile(self.cluster['profileId']) playbook = get_property( 'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK) playbook_params = get_property( 'config.launch.params', self.cluster, default={}) playbook_params['cluster_state'] = 'absent' cumulus.ansible.tasks.cluster.terminate_cluster \ .delay(playbook, self._model.filter(self.cluster, getCurrentUser(), passphrase=False), profile, secret_key, playbook_params, girder_token, log_write_url, ClusterStatus.TERMINATED)
def delete(self, volume, params): if 'clusterId' in volume: raise RestException('Unable to delete attached volume') # If the volume is in state created and it has no ec2 volume id # associated with it, we should be able to just delete it if volume['status'] == VolumeState.CREATED: if 'id' in volume['ec2'] and volume['ec2']['id'] is not None: raise RestException( 'Unable to delete volume, it is ' 'associated with an ec2 volume %s' % volume['ec2']['id']) self._model.remove(volume) return None # Call EC2 to delete volume profile_id = parse('profileId').find(volume)[0].value profile, secret_key = _get_profile(profile_id) girder_callback_info = { 'girder_api_url': getApiUrl(), 'girder_token': get_task_token()['_id']} p = CloudProvider(dict(secretAccessKey=secret_key, **profile)) aws_volume = p.get_volume(volume) if aws_volume['state'] != VolumeState.AVAILABLE: raise RestException( 'Volume must be in an "%s" status to be deleted' % VolumeState.AVAILABLE, 400) user = getCurrentUser() cumulus.ansible.tasks.volume.delete_volume\ .delay(profile, self._model.filter(volume, user), secret_key, girder_callback_info) volume['status'] = VolumeState.DELETING volume = self._model.update_volume(user, volume) return self._model.filter(volume, user)
def start(self, request_body): """ Adapters may implement this if they support a start operation. """ self.status = ClusterStatus.LAUNCHING self.cluster['config'].setdefault('provision', {})\ .setdefault('params', {}).update(request_body) self.cluster = self.model('cluster', 'cumulus').save(self.cluster) base_url = getApiUrl() log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id']) girder_token = get_task_token()['_id'] profile, secret_key = _get_profile(self.cluster['profileId']) # Launch launch_playbook = get_property( 'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK) launch_playbook_params = get_property( 'config.launch.params', self.cluster, default={}) launch_playbook_params['cluster_state'] = ClusterStatus.RUNNING # Provision provision_playbook = get_property( 'config.provision.spec', self.cluster, default='gridengine/site') provision_playbook_params = get_property( 'config.provision.params', self.cluster, default={}) provision_ssh_user = get_property( 'config.provision.ssh.user', self.cluster, default='ubuntu') provision_playbook_params['ansible_ssh_user'] = provision_ssh_user provision_playbook_params['cluster_state'] = ClusterStatus.RUNNING cumulus.ansible.tasks.cluster.start_cluster \ .delay(launch_playbook, # provision playbook provision_playbook, self._model.filter(self.cluster, getCurrentUser(), passphrase=False), profile, secret_key, launch_playbook_params, provision_playbook_params, girder_token, log_write_url)
def test_docker_run_transfer_encoding_stream(self, params): item_id = params.get('itemId') file_id = params.get('fileId') delimiter = params.get('delimiter') headers = { 'Girder-Token': str(Token().createToken(getCurrentUser())['_id']) } url = '%s/%s?itemId=%s&delimiter=%s' % ( getApiUrl(), 'integration_tests/docker/input_stream', item_id, delimiter) container_args = [ 'read_write', '-i', GirderFileIdToVolume(file_id), '-o', Connect(NamedOutputPipe('out'), ChunkedTransferEncodingStream(url, headers)) ] result = docker_run.delay( TEST_IMAGE, pull_image=True, container_args=container_args, remove_container=True) return result.job
def getUrl(cls, state): clientId = Setting().get(PluginSettings.GOOGLE_CLIENT_ID) if not clientId: raise Exception('No Google client ID setting is present.') callbackUrl = '/'.join((getApiUrl(), 'oauth', 'google', 'callback')) query = urllib.parse.urlencode({ 'response_type': 'code', 'access_type': 'online', 'client_id': clientId, 'redirect_uri': callbackUrl, 'state': state, 'scope': 'openid %s' % ' '.join(cls._AUTH_SCOPES) # Google supports a 'nonce', but does not require it, and Girder's particular checks # and invalidation of the 'state' token provides the same security guarantees }) return '%s?%s' % (cls._AUTH_URL, query)
def jobInfoSpec(job, token=None, logPrint=True): """ Build the jobInfo specification for a task to write status and log output back to a Girder job. :param job: The job document representing the worker task. :type job: dict :param token: The token to use. Creates a job token if not passed. :type token: str or dict :param logPrint: Whether standard output from the job should be """ if token is None: token = ModelImporter.model('job', 'jobs').createJobToken(job) if isinstance(token, dict): token = token['_id'] return { 'method': 'PUT', 'url': '/'.join((getApiUrl(), 'job', str(job['_id']))), 'reference': str(job['_id']), 'headers': {'Girder-Token': token}, 'logPrint': logPrint }
def terminate(self, id, params): (user, token) = self.getCurrentUser(returnToken=True) job = self._model.load(id, user=user, level=AccessType.ADMIN) if not job: raise RestException('Job not found.', code=404) cluster_model = self.model('cluster', 'cumulus') cluster = cluster_model.load(job['clusterId'], user=user, level=AccessType.ADMIN) base_url = getApiUrl() self._model.update_status(user, id, JobState.TERMINATING) log_url = '%s/jobs/%s/log' % (base_url, id) # Clean up job job = self._clean(job) girder_token = self.get_task_token()['_id'] tasks.job.terminate_job.delay(cluster, job, log_write_url=log_url, girder_token=girder_token) return job
def initMetrics(self, phase, params): user = self.getCurrentUser() apiUrl = getApiUrl() jobModel = self.model('job', 'jobs') title = '%s: metric weight initialization' % phase['name'] job = jobModel.createJob(title=title, type='covalic_weight_init', user=user, handler='worker_handler') jobToken = jobModel.createJobToken(job) scoreToken = self.model('token').createToken(user=user, days=7) groundTruth = self.model('folder').load(phase['groundTruthFolderId'], user=user, level=AccessType.READ, exc=True) kwargs = { 'task': { 'name': title, 'mode': 'docker', 'docker_image': 'girder/covalic-metrics:latest', 'container_args': [ '/covalic/Python/RankAggregation/computeWeights.py', '--groundtruth=$input{groundtruth}' ], 'entrypoint': 'python', 'inputs': [{ 'id': 'groundtruth', 'type': 'string', 'format': 'string', 'target': 'filepath', 'filename': 'groundtruth.zip' }], 'outputs': [{ 'id': '_stdout', 'format': 'string', 'type': 'string' }] }, 'inputs': { 'groundtruth': { 'mode': 'http', 'method': 'GET', 'url': '/'.join((apiUrl, 'folder', str(groundTruth['_id']), 'download')), 'headers': { 'Girder-Token': scoreToken['_id'] } } }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'string', 'url': '/'.join((apiUrl, 'challenge_phase', str(phase['_id']), 'metrics')), 'headers': { 'Girder-Token': scoreToken['_id'] } } }, 'jobInfo': { 'method': 'PUT', 'url': '/'.join((apiUrl, 'job', str(job['_id']))), 'headers': { 'Girder-Token': jobToken['_id'] }, 'logPrint': True }, 'validate': False, 'auto_convert': False, 'cleanup': True } job['kwargs'] = kwargs job = jobModel.save(job) jobModel.scheduleJob(job) return jobModel.filter(job, user)
def testGetApiUrl(self): url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test' self.assertEqual(rest.getApiUrl(url), 'https://localhost/thing/api/v1') url = 'https://localhost/girder#users' self.assertRaises(Exception, rest.getApiUrl, url=url)
def download_path(_id, resource): return "{}/{}/{}/download".format(getApiUrl(), resource, _id)
def runSlicerCliDescription(self, item, image, args, setName, setDescription, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % item['_id']) job = jobModel.createJob(title='Read docker Slicer CLI: %s' % image, type='item_task.slicer_cli', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] job.update({ 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((getApiUrl(), self.resourceName, str(item['_id']), 'slicer_cli_xml')), 'params': { 'setName': setName, 'setDescription': setDescription }, 'headers': { 'Girder-Token': token['_id'] } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False, 'cleanup': True } }) item['meta']['itemTaskSpec'] = { 'mode': 'docker', 'docker_image': image } if args: item['meta']['itemTaskSlicerCliArgs'] = args self.model('item').save(item) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def runClimos(self, inFolder, outFolder, params): self.requireParams(('seasons', 'vars', 'outputFilename'), params) user = self.getCurrentUser() urlParts = rest.getUrlParts() apiUrl = rest.getApiUrl() jobModel = self.model('job', 'jobs') job = jobModel.createJob(title='Climos: ' + inFolder['name'], type='climos', handler='romanesco_handler', user=user) token = self.model('token').createToken(user=user, days=3) task = { 'mode': 'python', 'script': _climos_script, 'inputs': [{ 'id': 'in_dir', 'type': 'string', 'format': 'text' }, { 'id': 'out_filename', 'type': 'string', 'format': 'text' }, { 'id': 'variables', 'type': 'python', 'format': 'object' }, { 'id': 'seasons', 'type': 'python', 'format': 'object' }], 'outputs': [{ 'id': 'outfile', 'type': 'string', 'format': 'text' }] } girderIoParams = { 'mode': 'girder', 'host': urlParts.hostname, 'port': urlParts.port, 'api_root': rest.getApiUrl(urlParts.path), 'scheme': urlParts.scheme, 'token': token['_id'] } inputs = { 'in_dir': dict( girderIoParams, **{ 'method': 'GET', 'id': str(inFolder['_id']), 'resource_type': 'folder', 'type': 'string', 'format': 'text', 'name': inFolder['name'] }), 'seasons': { 'mode': 'inline', 'type': 'python', 'format': 'object', 'data': json.loads(params['seasons']) }, 'variables': { 'mode': 'inline', 'type': 'python', 'format': 'object', 'data': json.loads(params['vars']) }, 'out_filename': { 'mode': 'inline', 'type': 'string', 'format': 'text', 'data': params['outputFilename'].strip() } } outputs = { 'outfile': dict( girderIoParams, **{ 'parent_type': 'folder', 'parent_id': str(outFolder['_id']), 'format': 'text', 'type': 'string' }) } job['kwargs'] = { 'task': task, 'inputs': inputs, 'outputs': outputs, 'jobInfo': { 'method': 'PUT', 'url': '/'.join((apiUrl, 'job', str(job['_id']))), 'headers': { 'Girder-Token': token['_id'] }, 'logPrint': True }, 'validate': False, 'auto_convert': True, 'cleanup': True } job = jobModel.save(job) jobModel.scheduleJob(job) return jobModel.filter(job, user)
def initMetrics(self, phase, params): user = self.getCurrentUser() apiUrl = getApiUrl() jobModel = self.model("job", "jobs") title = "%s: metric weight initialization" % phase["name"] job = jobModel.createJob(title=title, type="covalic_weight_init", user=user, handler="romanesco_handler") jobToken = jobModel.createJobToken(job) scoreToken = self.model("token").createToken(user=user, days=7) groundTruth = self.model("folder").load( phase["groundTruthFolderId"], user=user, level=AccessType.READ, exc=True ) kwargs = { "task": { "name": title, "mode": "docker", "docker_image": "girder/covalic-metrics:latest", "container_args": [ "/covalic/Python/RankAggregation/computeWeights.py", "--groundtruth=$input{groundtruth}", ], "entrypoint": "python", "inputs": [ { "id": "groundtruth", "type": "string", "format": "string", "target": "filepath", "filename": "groundtruth.zip", } ], "outputs": [{"id": "_stdout", "format": "string", "type": "string"}], }, "inputs": { "groundtruth": { "mode": "http", "method": "GET", "url": "/".join((apiUrl, "folder", str(groundTruth["_id"]), "download")), "headers": {"Girder-Token": scoreToken["_id"]}, } }, "outputs": { "_stdout": { "mode": "http", "method": "PUT", "format": "string", "url": "/".join((apiUrl, "challenge_phase", str(phase["_id"]), "metrics")), "headers": {"Girder-Token": scoreToken["_id"]}, } }, "jobInfo": { "method": "PUT", "url": "/".join((apiUrl, "job", str(job["_id"]))), "headers": {"Girder-Token": jobToken["_id"]}, "logPrint": True, }, "validate": False, "auto_convert": False, "cleanup": True, } job["kwargs"] = kwargs job = jobModel.save(job) jobModel.scheduleJob(job) return jobModel.filter(job, user)
def runClimos(self, inFolder, outFolder, params): self.requireParams(('seasons', 'vars', 'outputFilename'), params) user = self.getCurrentUser() urlParts = rest.getUrlParts() apiUrl = rest.getApiUrl() jobModel = self.model('job', 'jobs') job = jobModel.createJob( title='Climos: ' + inFolder['name'], type='climos', handler='romanesco_handler', user=user) token = self.model('token').createToken(user=user, days=3) task = { 'mode': 'python', 'script': _climos_script, 'inputs': [{ 'id': 'in_dir', 'type': 'string', 'format': 'text' }, { 'id': 'out_filename', 'type': 'string', 'format': 'text' }, { 'id': 'variables', 'type': 'python', 'format': 'object' }, { 'id': 'seasons', 'type': 'python', 'format': 'object' }], 'outputs': [{ 'id': 'outfile', 'type': 'string', 'format': 'text' }] } girderIoParams = { 'mode': 'girder', 'host': urlParts.hostname, 'port': urlParts.port, 'api_root': rest.getApiUrl(urlParts.path), 'scheme': urlParts.scheme, 'token': token['_id'] } inputs = { 'in_dir': dict(girderIoParams, **{ 'method': 'GET', 'id': str(inFolder['_id']), 'resource_type': 'folder', 'type': 'string', 'format': 'text', 'name': inFolder['name'] }), 'seasons': { 'mode': 'inline', 'type': 'python', 'format': 'object', 'data': json.loads(params['seasons']) }, 'variables': { 'mode': 'inline', 'type': 'python', 'format': 'object', 'data': json.loads(params['vars']) }, 'out_filename': { 'mode': 'inline', 'type': 'string', 'format': 'text', 'data': params['outputFilename'].strip() } } outputs = { 'outfile': dict(girderIoParams, **{ 'parent_type': 'folder', 'parent_id': str(outFolder['_id']), 'format': 'text', 'type': 'string' }) } job['kwargs'] = { 'task': task, 'inputs': inputs, 'outputs': outputs, 'jobInfo': { 'method': 'PUT', 'url': '/'.join((apiUrl, 'job', str(job['_id']))), 'headers': {'Girder-Token': token['_id']}, 'logPrint': True }, 'validate': False, 'auto_convert': True, 'cleanup': True } job = jobModel.save(job) jobModel.scheduleJob(job) return jobModel.filter(job, user)
def testGetApiUrl(self): url = "https://localhost/thing/api/v1/hello/world?foo=bar#test" self.assertEqual(rest.getApiUrl(url), "https://localhost/thing/api/v1") url = "https://localhost/girder#users" self.assertRaises(Exception, rest.getApiUrl, url=url)