def testGetApiUrl(self):
url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test'
self.assertEqual(rest.getApiUrl(url), 'https://localhost/thing/api/v1')
parts = rest.getUrlParts(url)
self.assertEqual(parts.path, '/thing/api/v1/hello/world')
self.assertEqual(rest.getApiUrl(parts.path), '/thing/api/v1')
self.assertEqual(parts.port, None)
self.assertEqual(parts.hostname, 'localhost')
self.assertEqual(parts.query, 'foo=bar')
self.assertEqual(parts.fragment, 'test')
url = 'https://localhost/girder#users'
self.assertRaises(Exception, rest.getApiUrl, url=url)
def testGetApiUrl(self):
url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test'
self.assertEqual(rest.getApiUrl(url), 'https://localhost/thing/api/v1')
parts = rest.getUrlParts(url)
self.assertEqual(parts.path, '/thing/api/v1/hello/world')
self.assertEqual(rest.getApiUrl(parts.path), '/thing/api/v1')
self.assertEqual(parts.port, None)
self.assertEqual(parts.hostname, 'localhost')
self.assertEqual(parts.query, 'foo=bar')
self.assertEqual(parts.fragment, 'test')
url = 'https://localhost/girder#users'
self.assertRaises(Exception, rest.getApiUrl, url=url)
def testGetApiUrl():
url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test'
assert rest.getApiUrl(url) == 'https://localhost/thing/api/v1'
parts = rest.getUrlParts(url)
assert parts.path == '/thing/api/v1/hello/world'
assert rest.getApiUrl(parts.path) == '/thing/api/v1'
assert parts.port is None
assert parts.hostname == 'localhost'
assert parts.query == 'foo=bar'
assert parts.fragment == 'test'
url = 'https://localhost/girder#users'
with pytest.raises(GirderException, match='Could not determine API root in %s.$' % url):
rest.getApiUrl(url)
def testGetApiUrl():
url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test'
assert rest.getApiUrl(url) == 'https://localhost/thing/api/v1'
parts = rest.getUrlParts(url)
assert parts.path == '/thing/api/v1/hello/world'
assert rest.getApiUrl(parts.path) == '/thing/api/v1'
assert parts.port is None
assert parts.hostname == 'localhost'
assert parts.query == 'foo=bar'
assert parts.fragment == 'test'
url = 'https://localhost/girder#users'
with pytest.raises(GirderException, match='Could not determine API root in %s.$' % url):
rest.getApiUrl(url)
def nearestNeighborIndex(item, user, descriptorSet):
"""
Get the nearest neighbor index from a given item and descriptor set.
:param item: Item to find the nn index from, usually the item that the
user is performing the nearest neighbors search on.
:param user: The owner of the .smqtk folder.
:param descriptorSet: The relevant descriptor set.
"""
folder = ModelImporter.model('folder')
_GirderDataElement = functools.partial(GirderDataElement,
api_root=getApiUrl(),
token=getCurrentToken()['_id'])
smqtkFolder = folder.createFolder(folder.load(item['folderId'], user=user), '.smqtk',
reuseExisting=True)
try:
meanVecFileId = localSmqtkFileIdFromName(smqtkFolder, 'mean_vec.npy')
rotationFileId = localSmqtkFileIdFromName(smqtkFolder, 'rotation.npy')
hash2uuidsFileId = localSmqtkFileIdFromName(smqtkFolder, 'hash2uuids.pickle')
except Exception:
logger.warn('SMQTK files didn\'t exist for performing NN on %s' % item['_id'])
return None
# TODO Should these be Girder data elements? Unnecessary HTTP requests.
functor = ItqFunctor(mean_vec_cache=_GirderDataElement(meanVecFileId),
rotation_cache=_GirderDataElement(rotationFileId))
hash2uuidsKV = MemoryKeyValueStore(_GirderDataElement(hash2uuidsFileId))
return LSHNearestNeighborIndex(functor, descriptorSet,
hash2uuidsKV, read_only=True)
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.SYNAPSE_CLIENT_ID)
if not clientId:
raise Exception('No Synapse client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'synapse', 'callback'))
# OIDC claims. The complete list is here:
# https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/oauth/OIDCClaimName.html
claims = {
'email': None,
'given_name': None,
'family_name': None
}
tokenAndUserInfoClaims = {'id_token': claims, 'userinfo': claims}
query = urllib.parse.urlencode({
'response_type': 'code',
'access_type': 'online',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': 'openid %s' % ' '.join(cls._AUTH_SCOPES),
'claims': json.dumps(tokenAndUserInfoClaims)
# Synapse supports a 'nonce', but does not require it, and Girder's particular checks
# and invalidation of the 'state' token provides the same security guarantees
})
return '%s?%s' % (cls._AUTH_URL, query)
def girderOutputSpec(parent, token, parentType='folder', name=None,
dataType='string', dataFormat='text'):
"""
Downstream plugins that are building worker jobs that use Girder IO
should use this to generate the output specs more easily.
:param parent: The parent to upload the data into (an item or folder).
:type parent: dict
:param token: The Girder token document or raw token string to use to
authenticate when uploading.
:type token: dict or str
:param parentType: The type of the parent object ("item" or "folder").
:type parentType: str
:param name: Name of the resource to use when uploading. Required if
the output target type is "memory". If the target is "filepath", uses
the basename of the file being uploaded by default.
:type name: str or None
:param dataType: The worker `type` field.
:type dataType: str
:param dataFormat: The worker `format` field.
:type dataFormat: str
"""
if isinstance(token, dict):
token = token['_id']
return {
'mode': 'girder',
'api_url': getApiUrl(),
'token': token,
'name': name,
'parent_id': str(parent['_id']),
'parent_type': parentType,
'type': dataType,
'format': dataFormat
}
def girderInputSpec(resource, resourceType='file', name=None, token=None,
dataType='string', dataFormat='text'):
"""
Downstream plugins that are building Girder worker jobs that use Girder IO
should use this to generate the input specs more easily.
:param resource: The resource document to be downloaded at runtime.
:type resource: dict
:param resourceType: The resource type to download for the input. Should
be "folder", "item", or "file".
:type resourceType: str
:param name: The name of the resource to download. If not passed, uses
the "name" field of the resource document.
:type name: str or None
:param token: The Girder token document or raw token string to use to
authenticate when downloading. Pass `None` for anonymous downloads.
:type token: dict, str, or None
:param dataType: The worker `type` field.
:type dataType: str
:param dataFormat: The worker `format` field.
:type dataFormat: str
"""
if isinstance(token, dict):
token = token['_id']
return {
'mode': 'girder',
'api_url': getApiUrl(),
'token': token,
'id': str(resource['_id']),
'name': name or resource['name'],
'resource_type': resourceType,
'type': dataType,
'format': dataFormat
}
def testStream(self, item, params):
token = self.getCurrentToken()
jobModel = self.model('job', 'jobs')
job = jobModel.createJob(title='docker stream test',
type='docker_test',
handler='worker_handler',
user=self.getCurrentUser())
jobToken = jobModel.createJobToken(job)
apiUrl = getApiUrl()
kwargs = {
'task': {
'mode':
'docker',
'docker_image':
'testoutputs:latest',
'pull_image':
False,
'inputs': [{
'id': 'input_pipe',
'target': 'filepath',
'stream': True
}],
'outputs': [{
'id': 'output_pipe',
'target': 'filepath',
'stream': True
}]
},
'inputs': {
'input_pipe': {
'mode': 'http',
'method': 'GET',
'url': '%s/item/%s/download' % (apiUrl, str(item['_id'])),
'headers': {
'Girder-Token': str(token['_id'])
}
}
},
'outputs': {
'output_pipe': {
'mode': 'http',
'method': 'POST',
'url': apiUrl + '/docker_test/stream_callback',
'headers': {
'Girder-Token': str(token['_id'])
}
}
},
'validate': False,
'auto_convert': False,
'cleanup': False,
'jobInfo': utils.jobInfoSpec(job, jobToken)
}
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def jobInfoSpec(job, token=None, logPrint=True):
"""
Build the jobInfo specification for a task to write status and log output
back to a Girder job.
:param job: The job document representing the worker task.
:type job: dict
:param token: The token to use. Creates a job token if not passed.
:type token: str or dict
:param logPrint: Whether standard output from the job should be
"""
if token is None:
token = ModelImporter.model('job', 'jobs').createJobToken(job)
if isinstance(token, dict):
token = token['_id']
return {
'method': 'PUT',
'url': '/'.join((getApiUrl(), 'job', str(job['_id']))),
'reference': str(job['_id']),
'headers': {
'Girder-Token': token
},
'logPrint': logPrint
}
def testServerRootSetting(self):
settingModel = Setting()
with self.assertRaises(ValidationException):
settingModel.set(SettingKey.SERVER_ROOT, 'bad_value')
settingModel.set(SettingKey.SERVER_ROOT, 'https://somedomain.org/foo')
self.assertEqual(getApiUrl(), 'https://somedomain.org/foo/api/v1')
def testServerRootSetting(self):
settingModel = self.model('setting')
with self.assertRaises(ValidationException):
settingModel.set(SettingKey.SERVER_ROOT, 'bad_value')
settingModel.set(SettingKey.SERVER_ROOT, 'https://somedomain.org/foo')
self.assertEqual(getApiUrl(), 'https://somedomain.org/foo/api/v1')
def nearestNeighborIndex(item, user, descriptorIndex):
"""
Get the nearest neighbor index from a given item and descriptor index.
:param item: Item to find the nn index from, usually the item that the
user is performing the nearest neighbors search on.
:param user: The owner of the .smqtk folder.
:param descriptorIndex: The relevant descriptor index.
"""
folder = ModelImporter.model('folder')
_GirderDataElement = functools.partial(GirderDataElement,
api_root=getApiUrl(),
token=getCurrentToken()['_id'])
smqtkFolder = folder.createFolder(folder.load(item['folderId'], user=user), '.smqtk',
reuseExisting=True)
try:
meanVecFileId = localSmqtkFileIdFromName(smqtkFolder, 'mean_vec.npy')
rotationFileId = localSmqtkFileIdFromName(smqtkFolder, 'rotation.npy')
hash2uuidsFileId = localSmqtkFileIdFromName(smqtkFolder, 'hash2uuids.pickle')
except Exception:
logger.warn('SMQTK files didn\'t exist for performing NN on %s' % item['_id'])
return None
# TODO Should these be Girder data elements? Unnecessary HTTP requests.
functor = ItqFunctor(mean_vec_cache=_GirderDataElement(meanVecFileId),
rotation_cache=_GirderDataElement(rotationFileId))
hash2uuidsKV = MemoryKeyValueStore(_GirderDataElement(hash2uuidsFileId))
return LSHNearestNeighborIndex(functor, descriptorIndex,
hash2uuidsKV, read_only=True)
def provision(self):
self.status = ClusterStatus.PROVISIONING
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property(
'config.provision.spec', self.cluster,
default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property(
'config.provision.params', self.cluster, default={})
provision_ssh_user = get_property(
'config.provision.ssh.user', self.cluster, default='ubuntu')
playbook_params['cluster_state'] = ClusterStatus.RUNNING
playbook_params['ansible_ssh_user'] = provision_ssh_user
cumulus.ansible.tasks.cluster.provision_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params,
girder_token, log_write_url, ClusterStatus.RUNNING)
return self.cluster
def submit_job(self, job):
log_url = '%s/jobs/%s/log' % (getApiUrl(), job['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.job.submit(
girder_token,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
job, log_url)
def getWorkerApiUrl():
"""
Return the API base URL to which the worker should callback to
write output information back to the server. This is controlled
via a system setting, and the default is to use the core server
root setting.
"""
apiUrl = ModelImporter.model('setting').get(PluginSettings.API_URL)
return apiUrl or getApiUrl()
def getWorkerApiUrl():
"""
Return the API base URL to which the worker should callback to
write output information back to the server. This is controlled
via a system setting, and the default is to use the core server
root setting.
"""
apiUrl = Setting().get(PluginSettings.API_URL)
return apiUrl or getApiUrl()
def start(self, request_body):
log_write_url = '%s/clusters/%s/log' % (getApiUrl(),
self.cluster['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.cluster.test_connection \
.delay(
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url, girder_token=girder_token)
def createBlurImage(self, item, params):
user = self.getCurrentUser()
token = self.getCurrentToken()
jobTitle = 'ITK blur: ' + item['name']
jobModel = self.model('job', 'jobs')
folder = self.model('folder').load(item['folderId'], force=True)
job = jobModel.createJob(
title=jobTitle, type='itk_blur', handler='worker_handler',
user=user)
jobToken = jobModel.createJobToken(job)
scriptFile = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'scripts', 'cad_script.py')
with open(scriptFile, 'r') as fh:
script = fh.read()
kwargs = {
'task': {
'name': jobTitle,
'mode': 'python',
'script': script,
'inputs': [{
'id': 'inputFileName',
'type': 'string',
'format': 'text',
'target': 'filepath'
}],
'outputs': [{
'id': 'outputFileName',
'format': 'text',
'type': 'string',
'target': 'filepath'
}]
},
'inputs': {
'inputFileName': utils.girderInputSpec(
item, resourceType='item', token=token)
},
'outputs': {
'outputFileName': utils.girderOutputSpec(
folder, token=token, parentType='folder')
},
'jobInfo': {
'method': 'PUT',
'url': '/'.join((getApiUrl(), 'job', str(job['_id']))),
'headers': {'Girder-Token': jobToken['_id']},
'logPrint': True
},
'validate': False,
'auto_convert': False
}
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def testStream(self, item, params):
token = self.getCurrentToken()
jobModel = self.model('job', 'jobs')
job = jobModel.createJob(
title='docker stream test', type='docker_test',
handler='worker_handler', user=self.getCurrentUser())
jobToken = jobModel.createJobToken(job)
apiUrl = getApiUrl()
kwargs = {
'task': {
'mode': 'docker',
'docker_image': 'testoutputs:latest',
'pull_image': False,
'inputs': [{
'id': 'input_pipe',
'target': 'filepath',
'stream': True
}],
'outputs': [{
'id': 'output_pipe',
'target': 'filepath',
'stream': True
}]
},
'inputs': {
'input_pipe': {
'mode': 'http',
'method': 'GET',
'url': '%s/item/%s/download' % (apiUrl, str(item['_id'])),
'headers': {
'Girder-Token': str(token['_id'])
}
}
},
'outputs': {
'output_pipe': {
'mode': 'http',
'method': 'POST',
'url': apiUrl + '/docker_test/stream_callback',
'headers': {
'Girder-Token': str(token['_id'])
}
}
},
'validate': False,
'auto_convert': False,
'cleanup': False,
'jobInfo': utils.jobInfoSpec(job, jobToken)
}
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def _startContainer(self, container):
settings = Setting()
psRoot = settings.get(PluginSettings.PRIVATE_STORAGE_PATH)
restUrl = rest.getApiUrl()
token = rest.getCurrentToken()['_id']
sessionId = str(container['sessionId'])
mountId = efs.mount(sessionId, '/tmp/' + sessionId, psRoot, restUrl, token)
container['mountId'] = mountId
container['status'] = 'Running'
self.save(container)
def getUrl(cls, state):
_, _, redirect = state.partition(".")
url = "/".join((getApiUrl(), "account",
cls.getProviderName(external=False), "callback"))
url += "?state={}&code=dataone".format(quote(state))
auth_url = urlparse(cls.get_cn())._replace(
path="/portal/oauth",
query="action=start&target={}".format(quote(url)))
return urlunparse(auth_url)
def runJsonTasksDescription(self, folder, image, pullImage, params):
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3,
scope='item_task.set_task_spec.%s' % folder['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(title='Read docker task specs: %s' % image,
type='item_task.json_description',
handler='worker_handler',
user=self.getCurrentUser())
jobOptions = {
'itemTaskId': folder['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': [],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'POST',
'format':
'text',
'url':
'/'.join((getApiUrl(), self.resourceName,
str(folder['_id']), 'json_specs')),
'headers': {
'Girder-Token': token['_id']
},
'params': {
'image': image,
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False,
'cleanup': True
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def createInstance(self, tale, user, token, name=None, save=True):
existing = self.findOne({
'taleId': tale['_id'],
'creatorId': user['_id'],
})
if existing:
return existing
if not name:
name = tale.get('title', '')
workspaceFolder = self.model('tale', 'wholetale').createWorkspace(tale)
now = datetime.datetime.utcnow()
payload = {
'girder_token': str(token['_id']),
'apiUrl': getWorkerApiUrl(),
'taleId': str(tale['_id']),
'workspaceId': str(workspaceFolder['_id']),
'api_version': API_VERSION
}
volumeTask = getCeleryApp().send_task('gwvolman.tasks.create_volume',
args=[payload])
volume = volumeTask.get(timeout=TASK_TIMEOUT)
payload.update(volume)
serviceTask = getCeleryApp().send_task(
'gwvolman.tasks.launch_container', args=[payload], queue='manager')
service = serviceTask.get(timeout=TASK_TIMEOUT)
service.update(volume)
netloc = urllib.parse.urlsplit(getApiUrl()).netloc
domain = '{}.{}'.format(service['name'],
netloc.split(':')[0].split('.', 1)[1])
url = 'https://{}/{}'.format(domain, service.get('urlPath', ''))
_wait_for_server(url)
instance = {
'taleId': tale['_id'],
'created': now,
'creatorId': user['_id'],
'lastActivity': now,
'containerInfo': service,
'name': name,
'status': InstanceStatus.RUNNING, # be optimistic for now
'url': url
}
self.setUserAccess(instance, user=user, level=AccessType.ADMIN)
if save:
instance = self.save(instance)
return instance
def start(self, request_body):
if self.cluster['status'] == ClusterStatus.CREATING:
raise RestException('Cluster is not ready to start.', code=400)
log_write_url = '%s/clusters/%s/log' % (getApiUrl(),
self.cluster['_id'])
girder_token = get_task_token()['_id']
cumulus.tasks.cluster.test_connection \
.delay(self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url,
girder_token=girder_token)
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.GITHUB_CLIENT_ID)
if not clientId:
raise Exception('No GitHub client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'github', 'callback'))
query = urllib.parse.urlencode({
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ','.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.BOX_CLIENT_ID)
if not clientId:
raise Exception('No Box client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'box', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
})
return '%s?%s' % (cls._AUTH_URL, query)
def process(self, workingSet, options, params):
"""
Run the complete processing workflow.
"""
user = self.getCurrentUser()
apiUrl = getApiUrl()
token = getCurrentToken()
outputFolder = self._outputFolder(workingSet)
requestInfo = RequestInfo(user=user, apiUrl=apiUrl, token=token)
workflowManager = DanesfieldWorkflowManager.instance()
jobId = workflowManager.initJob(requestInfo, workingSet, outputFolder,
options)
workflowManager.advance(jobId=jobId)
def getUrl(cls, state):
clientId = Setting().get(constants.PluginSettings.GITHUB_CLIENT_ID)
if clientId is None:
raise Exception('No GitHub client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'github', 'callback'))
query = urllib.parse.urlencode({
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ','.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.BOX_CLIENT_ID)
if not clientId:
raise Exception('No Box client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'box', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
})
return '%s?%s' % (cls._AUTH_URL, query)
def _testSettings(self, providerInfo):
Setting().set(SettingKey.REGISTRATION_POLICY, 'closed')
self.accountType = 'new'
# We should get an empty listing when no providers are set up
params = {'key': PluginSettings.PROVIDERS_ENABLED, 'value': []}
resp = self.request('/system/setting',
user=self.adminUser,
method='PUT',
params=params)
self.assertStatusOk(resp)
resp = self.request('/nciLogin/loginCallback', exception=True)
self.assertStatusOk(resp)
self.assertFalse(resp.json)
# Set up provider normally
params = {
'list':
json.dumps([{
'key': PluginSettings.PROVIDERS_ENABLED,
'value': [providerInfo['id']]
}, {
'key': PluginSettings.NCI_CLIENT_ID,
'value': providerInfo['id']
}, {
'key': PluginSettings.NCI_RETURN_URL,
'value': providerInfo['return_url']
}, {
'key': PluginSettings.NCI_LOGIN_URL,
'value': providerInfo['login_url']
}, {
'key': PluginSettings.NCI_VALIDATION_URL,
'value': providerInfo['validation_url']
}])
}
resp = self.request('/system/setting',
user=self.adminUser,
method='PUT',
params=params)
self.assertStatusOk(resp)
# # No need to re-fetch and test all of these settings values; they will
# # be implicitly tested later
resp = self.request('/nciLogin/loginCallback', exception=True)
self.assertStatusOk(resp)
expect = 'https://bar.com?returnUrl=' + '/'.join(
(getApiUrl(), 'nciLogin', 'callback'))
self.assertEqual(resp.json, expect)
def getUrl(cls, state):
clientId = Setting().get(constants.PluginSettings.BITBUCKET_CLIENT_ID)
if clientId is None:
raise Exception('No Bitbucket client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'bitbucket', 'callback'))
query = urllib.parse.urlencode({
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'response_type': 'code',
'scope': ','.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state):
clientId = Setting().get(constants.PluginSettings.LINKEDIN_CLIENT_ID)
if clientId is None:
raise Exception('No LinkedIn client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'linkedin', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ' '.join(cls._AUTH_SCOPES)
})
return '?'.join((cls._AUTH_URL, query))
def getUrl(cls, state):
clientId = Setting().get(constants.PluginSettings.GLOBUS_CLIENT_ID)
if clientId is None:
raise Exception('No Globus client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'globus', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'access_type': 'online',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ' '.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state):
clientId = cls.model('setting').get(
constants.PluginSettings.BITBUCKET_CLIENT_ID)
if clientId is None:
raise Exception('No Bitbucket client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'bitbucket', 'callback'))
query = urllib.parse.urlencode({
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'response_type': 'code',
'scope': ','.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.GLOBUS_CLIENT_ID)
if not clientId:
raise Exception('No Globus client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'globus', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'access_type': 'online',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ' '.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def _emitHook(event, hook):
body = json.dumps(
{
'apiUrl': getApiUrl(),
'name': event.name,
'info': event.info,
'uid': uuid.uuid4()
},
cls=JsonEncoder)
headers = {'Content-Type': 'application/json'}
if 'hmacKey' in hook:
headers['Girder-Signature'] = 'sha256=' + hmac.new(
hook['hmacKey'].encode('utf8'), body, hashlib.sha256).hexdigest()
# TODO this is blocking (because requests is blocking). We should do this asynchronously.
requests.post(hook['url'], data=body, headers=headers)
def girderOutputSpec(parent,
token,
parentType='folder',
name=None,
dataType='string',
dataFormat='text',
reference=None):
"""
Downstream plugins that are building worker jobs that use Girder IO
should use this to generate the output specs more easily.
:param parent: The parent to upload the data into (an item or folder).
:type parent: dict
:param token: The Girder token document or raw token string to use to
authenticate when uploading.
:type token: dict or str
:param parentType: The type of the parent object ("item" or "folder").
:type parentType: str
:param name: Name of the resource to use when uploading. Required if
the output target type is "memory". If the target is "filepath", uses
the basename of the file being uploaded by default.
:type name: str or None
:param dataType: The worker `type` field.
:type dataType: str
:param dataFormat: The worker `format` field.
:type dataFormat: str
:param reference: Optional "reference" string to pass back to the server
during the upload. This can be used to attach arbitrary data to this
for tracking purposes, e.g., referring back to related inputs.
:type reference: str
"""
if isinstance(token, dict):
token = token['_id']
return {
'mode': 'girder',
'api_url': getApiUrl(),
'token': token,
'name': name,
'parent_id': str(parent['_id']),
'parent_type': parentType,
'type': dataType,
'format': dataFormat,
'reference': reference
}
def getUrl(cls, state):
clientId = cls.model('setting').get(
constants.PluginSettings.GOOGLE_CLIENT_ID)
if clientId is None:
raise Exception('No Google client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'google', 'callback'))
query = urllib.parse.urlencode({
'response_type': 'code',
'access_type': 'online',
'client_id': clientId,
'redirect_uri': callbackUrl,
'state': state,
'scope': ' '.join(cls._AUTH_SCOPES)
})
return '%s?%s' % (cls._AUTH_URL, query)
def detach(self, volume, params):
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume is None or aws_volume['state'] != VolumeState.INUSE:
raise RestException('This volume is not attached '
'to a cluster',
400)
if 'clusterId' not in volume:
raise RestException('clusterId is not set on this volume!', 400)
try:
volume['path']
except KeyError:
raise RestException('path is not set on this volume!', 400)
cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
user=getCurrentUser(),
level=AccessType.ADMIN)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!',
400)
user = getCurrentUser()
cluster = self.model('cluster', 'cumulus').filter(
cluster, user, passphrase=False)
cumulus.ansible.tasks.volume.detach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, user),
secret_key, girder_callback_info)
volume['status'] = VolumeState.DETACHING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def girderInputSpec(resource,
resourceType='file',
name=None,
token=None,
dataType='string',
dataFormat='text',
fetchParent=False):
"""
Downstream plugins that are building Girder worker jobs that use Girder IO
should use this to generate the input specs more easily.
:param resource: The resource document to be downloaded at runtime.
:type resource: dict
:param resourceType: The resource type to download for the input. Should
be "folder", "item", or "file".
:type resourceType: str
:param name: The name of the resource to download. If not passed, uses
the "name" field of the resource document.
:type name: str or None
:param token: The Girder token document or raw token string to use to
authenticate when downloading. Pass `None` for anonymous downloads.
:type token: dict, str, or None
:param dataType: The worker `type` field.
:type dataType: str
:param dataFormat: The worker `format` field.
:type dataFormat: str
:param fetchParent: Whether to fetch the whole parent resource of the
specified resource as a side effect.
:type fetchParent: bool
"""
if isinstance(token, dict):
token = token['_id']
return {
'mode': 'girder',
'api_url': getApiUrl(),
'token': token,
'id': str(resource['_id']),
'name': name or resource['name'],
'resource_type': resourceType,
'type': dataType,
'format': dataFormat,
'fetch_parent': fetchParent
}
def attach(self, volume, cluster, params):
body = getBodyJson()
self.requireParams(['path'], body)
path = body['path']
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
# If volume exists it needs to be available to be attached. If
# it doesn't exist it will be created as part of the attach
# playbook.
if aws_volume is not None and \
aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException('This volume is not available to attach '
'to a cluster',
400)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!',
400)
cluster = self.model('cluster', 'cumulus').filter(
cluster, getCurrentUser(), passphrase=False)
cumulus.ansible.tasks.volume.attach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, getCurrentUser()), path,
secret_key, girder_callback_info)
volume['status'] = VolumeState.ATTACHING
volume = self._model.update_volume(getCurrentUser(), volume)
return self._model.filter(volume, getCurrentUser())
def jobInfoSpec(job, token, logPrint=True):
"""
Build the jobInfo specification for a romanesco task to write status and
log output back to a Girder job.
:param job: The job document representing the romanesco task.
:type job: dict
:param token: The token
:type token: str or dict
:param logPrint: Whether standard output from the job should be
"""
if type(token) is dict:
token = token['_id']
return {
'method': 'PUT',
'url': '/'.join((getApiUrl(), 'job', str(job['_id']))),
'headers': {'Girder-Token': token},
'logPrint': logPrint
}
def terminate(self):
self.status = ClusterStatus.TERMINATING
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property(
'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property(
'config.launch.params', self.cluster, default={})
playbook_params['cluster_state'] = 'absent'
cumulus.ansible.tasks.cluster.terminate_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params, girder_token,
log_write_url, ClusterStatus.TERMINATED)
def delete(self, volume, params):
if 'clusterId' in volume:
raise RestException('Unable to delete attached volume')
# If the volume is in state created and it has no ec2 volume id
# associated with it, we should be able to just delete it
if volume['status'] == VolumeState.CREATED:
if 'id' in volume['ec2'] and volume['ec2']['id'] is not None:
raise RestException(
'Unable to delete volume, it is '
'associated with an ec2 volume %s' % volume['ec2']['id'])
self._model.remove(volume)
return None
# Call EC2 to delete volume
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'Volume must be in an "%s" status to be deleted'
% VolumeState.AVAILABLE, 400)
user = getCurrentUser()
cumulus.ansible.tasks.volume.delete_volume\
.delay(profile, self._model.filter(volume, user),
secret_key, girder_callback_info)
volume['status'] = VolumeState.DELETING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def start(self, request_body):
"""
Adapters may implement this if they support a start operation.
"""
self.status = ClusterStatus.LAUNCHING
self.cluster['config'].setdefault('provision', {})\
.setdefault('params', {}).update(request_body)
self.cluster = self.model('cluster', 'cumulus').save(self.cluster)
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
# Launch
launch_playbook = get_property(
'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK)
launch_playbook_params = get_property(
'config.launch.params', self.cluster, default={})
launch_playbook_params['cluster_state'] = ClusterStatus.RUNNING
# Provision
provision_playbook = get_property(
'config.provision.spec', self.cluster, default='gridengine/site')
provision_playbook_params = get_property(
'config.provision.params', self.cluster, default={})
provision_ssh_user = get_property(
'config.provision.ssh.user', self.cluster, default='ubuntu')
provision_playbook_params['ansible_ssh_user'] = provision_ssh_user
provision_playbook_params['cluster_state'] = ClusterStatus.RUNNING
cumulus.ansible.tasks.cluster.start_cluster \
.delay(launch_playbook,
# provision playbook
provision_playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key,
launch_playbook_params, provision_playbook_params,
girder_token, log_write_url)
def test_docker_run_transfer_encoding_stream(self, params):
item_id = params.get('itemId')
file_id = params.get('fileId')
delimiter = params.get('delimiter')
headers = {
'Girder-Token': str(Token().createToken(getCurrentUser())['_id'])
}
url = '%s/%s?itemId=%s&delimiter=%s' % (
getApiUrl(), 'integration_tests/docker/input_stream', item_id, delimiter)
container_args = [
'read_write',
'-i', GirderFileIdToVolume(file_id),
'-o', Connect(NamedOutputPipe('out'),
ChunkedTransferEncodingStream(url, headers))
]
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=container_args,
remove_container=True)
return result.job
def getUrl(cls, state):
clientId = Setting().get(PluginSettings.GOOGLE_CLIENT_ID)
if not clientId:
raise Exception('No Google client ID setting is present.')
callbackUrl = '/'.join((getApiUrl(), 'oauth', 'google', 'callback'))
query = urllib.parse.urlencode({
'response_type':
'code',
'access_type':
'online',
'client_id':
clientId,
'redirect_uri':
callbackUrl,
'state':
state,
'scope':
'openid %s' % ' '.join(cls._AUTH_SCOPES)
# Google supports a 'nonce', but does not require it, and Girder's particular checks
# and invalidation of the 'state' token provides the same security guarantees
})
return '%s?%s' % (cls._AUTH_URL, query)
def jobInfoSpec(job, token=None, logPrint=True):
"""
Build the jobInfo specification for a task to write status and log output
back to a Girder job.
:param job: The job document representing the worker task.
:type job: dict
:param token: The token to use. Creates a job token if not passed.
:type token: str or dict
:param logPrint: Whether standard output from the job should be
"""
if token is None:
token = ModelImporter.model('job', 'jobs').createJobToken(job)
if isinstance(token, dict):
token = token['_id']
return {
'method': 'PUT',
'url': '/'.join((getApiUrl(), 'job', str(job['_id']))),
'reference': str(job['_id']),
'headers': {'Girder-Token': token},
'logPrint': logPrint
}
def terminate(self, id, params):
(user, token) = self.getCurrentUser(returnToken=True)
job = self._model.load(id, user=user, level=AccessType.ADMIN)
if not job:
raise RestException('Job not found.', code=404)
cluster_model = self.model('cluster', 'cumulus')
cluster = cluster_model.load(job['clusterId'], user=user,
level=AccessType.ADMIN)
base_url = getApiUrl()
self._model.update_status(user, id, JobState.TERMINATING)
log_url = '%s/jobs/%s/log' % (base_url, id)
# Clean up job
job = self._clean(job)
girder_token = self.get_task_token()['_id']
tasks.job.terminate_job.delay(cluster, job, log_write_url=log_url,
girder_token=girder_token)
return job
def initMetrics(self, phase, params):
user = self.getCurrentUser()
apiUrl = getApiUrl()
jobModel = self.model('job', 'jobs')
title = '%s: metric weight initialization' % phase['name']
job = jobModel.createJob(title=title,
type='covalic_weight_init',
user=user,
handler='worker_handler')
jobToken = jobModel.createJobToken(job)
scoreToken = self.model('token').createToken(user=user, days=7)
groundTruth = self.model('folder').load(phase['groundTruthFolderId'],
user=user,
level=AccessType.READ,
exc=True)
kwargs = {
'task': {
'name':
title,
'mode':
'docker',
'docker_image':
'girder/covalic-metrics:latest',
'container_args': [
'/covalic/Python/RankAggregation/computeWeights.py',
'--groundtruth=$input{groundtruth}'
],
'entrypoint':
'python',
'inputs': [{
'id': 'groundtruth',
'type': 'string',
'format': 'string',
'target': 'filepath',
'filename': 'groundtruth.zip'
}],
'outputs': [{
'id': '_stdout',
'format': 'string',
'type': 'string'
}]
},
'inputs': {
'groundtruth': {
'mode':
'http',
'method':
'GET',
'url':
'/'.join((apiUrl, 'folder', str(groundTruth['_id']),
'download')),
'headers': {
'Girder-Token': scoreToken['_id']
}
}
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'PUT',
'format':
'string',
'url':
'/'.join((apiUrl, 'challenge_phase', str(phase['_id']),
'metrics')),
'headers': {
'Girder-Token': scoreToken['_id']
}
}
},
'jobInfo': {
'method': 'PUT',
'url': '/'.join((apiUrl, 'job', str(job['_id']))),
'headers': {
'Girder-Token': jobToken['_id']
},
'logPrint': True
},
'validate': False,
'auto_convert': False,
'cleanup': True
}
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
return jobModel.filter(job, user)
def testGetApiUrl(self):
url = 'https://localhost/thing/api/v1/hello/world?foo=bar#test'
self.assertEqual(rest.getApiUrl(url), 'https://localhost/thing/api/v1')
url = 'https://localhost/girder#users'
self.assertRaises(Exception, rest.getApiUrl, url=url)
def download_path(_id, resource):
return "{}/{}/{}/download".format(getApiUrl(), resource, _id)
def runSlicerCliDescription(self, item, image, args, setName,
setDescription, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec',
{}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3, scope='item_task.set_task_spec.%s' % item['_id'])
job = jobModel.createJob(title='Read docker Slicer CLI: %s' % image,
type='item_task.slicer_cli',
handler='worker_handler',
user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
job.update({
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'PUT',
'format':
'text',
'url':
'/'.join((getApiUrl(), self.resourceName,
str(item['_id']), 'slicer_cli_xml')),
'params': {
'setName': setName,
'setDescription': setDescription
},
'headers': {
'Girder-Token': token['_id']
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False,
'cleanup': True
}
})
item['meta']['itemTaskSpec'] = {
'mode': 'docker',
'docker_image': image
}
if args:
item['meta']['itemTaskSlicerCliArgs'] = args
self.model('item').save(item)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def runClimos(self, inFolder, outFolder, params):
self.requireParams(('seasons', 'vars', 'outputFilename'), params)
user = self.getCurrentUser()
urlParts = rest.getUrlParts()
apiUrl = rest.getApiUrl()
jobModel = self.model('job', 'jobs')
job = jobModel.createJob(title='Climos: ' + inFolder['name'],
type='climos',
handler='romanesco_handler',
user=user)
token = self.model('token').createToken(user=user, days=3)
task = {
'mode':
'python',
'script':
_climos_script,
'inputs': [{
'id': 'in_dir',
'type': 'string',
'format': 'text'
}, {
'id': 'out_filename',
'type': 'string',
'format': 'text'
}, {
'id': 'variables',
'type': 'python',
'format': 'object'
}, {
'id': 'seasons',
'type': 'python',
'format': 'object'
}],
'outputs': [{
'id': 'outfile',
'type': 'string',
'format': 'text'
}]
}
girderIoParams = {
'mode': 'girder',
'host': urlParts.hostname,
'port': urlParts.port,
'api_root': rest.getApiUrl(urlParts.path),
'scheme': urlParts.scheme,
'token': token['_id']
}
inputs = {
'in_dir':
dict(
girderIoParams, **{
'method': 'GET',
'id': str(inFolder['_id']),
'resource_type': 'folder',
'type': 'string',
'format': 'text',
'name': inFolder['name']
}),
'seasons': {
'mode': 'inline',
'type': 'python',
'format': 'object',
'data': json.loads(params['seasons'])
},
'variables': {
'mode': 'inline',
'type': 'python',
'format': 'object',
'data': json.loads(params['vars'])
},
'out_filename': {
'mode': 'inline',
'type': 'string',
'format': 'text',
'data': params['outputFilename'].strip()
}
}
outputs = {
'outfile':
dict(
girderIoParams, **{
'parent_type': 'folder',
'parent_id': str(outFolder['_id']),
'format': 'text',
'type': 'string'
})
}
job['kwargs'] = {
'task': task,
'inputs': inputs,
'outputs': outputs,
'jobInfo': {
'method': 'PUT',
'url': '/'.join((apiUrl, 'job', str(job['_id']))),
'headers': {
'Girder-Token': token['_id']
},
'logPrint': True
},
'validate': False,
'auto_convert': True,
'cleanup': True
}
job = jobModel.save(job)
jobModel.scheduleJob(job)
return jobModel.filter(job, user)
def download_path(_id, resource):
return "{}/{}/{}/download".format(getApiUrl(), resource, _id)
def initMetrics(self, phase, params):
user = self.getCurrentUser()
apiUrl = getApiUrl()
jobModel = self.model("job", "jobs")
title = "%s: metric weight initialization" % phase["name"]
job = jobModel.createJob(title=title, type="covalic_weight_init", user=user, handler="romanesco_handler")
jobToken = jobModel.createJobToken(job)
scoreToken = self.model("token").createToken(user=user, days=7)
groundTruth = self.model("folder").load(
phase["groundTruthFolderId"], user=user, level=AccessType.READ, exc=True
)
kwargs = {
"task": {
"name": title,
"mode": "docker",
"docker_image": "girder/covalic-metrics:latest",
"container_args": [
"/covalic/Python/RankAggregation/computeWeights.py",
"--groundtruth=$input{groundtruth}",
],
"entrypoint": "python",
"inputs": [
{
"id": "groundtruth",
"type": "string",
"format": "string",
"target": "filepath",
"filename": "groundtruth.zip",
}
],
"outputs": [{"id": "_stdout", "format": "string", "type": "string"}],
},
"inputs": {
"groundtruth": {
"mode": "http",
"method": "GET",
"url": "/".join((apiUrl, "folder", str(groundTruth["_id"]), "download")),
"headers": {"Girder-Token": scoreToken["_id"]},
}
},
"outputs": {
"_stdout": {
"mode": "http",
"method": "PUT",
"format": "string",
"url": "/".join((apiUrl, "challenge_phase", str(phase["_id"]), "metrics")),
"headers": {"Girder-Token": scoreToken["_id"]},
}
},
"jobInfo": {
"method": "PUT",
"url": "/".join((apiUrl, "job", str(job["_id"]))),
"headers": {"Girder-Token": jobToken["_id"]},
"logPrint": True,
},
"validate": False,
"auto_convert": False,
"cleanup": True,
}
job["kwargs"] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
return jobModel.filter(job, user)
def runClimos(self, inFolder, outFolder, params):
self.requireParams(('seasons', 'vars', 'outputFilename'), params)
user = self.getCurrentUser()
urlParts = rest.getUrlParts()
apiUrl = rest.getApiUrl()
jobModel = self.model('job', 'jobs')
job = jobModel.createJob(
title='Climos: ' + inFolder['name'], type='climos',
handler='romanesco_handler', user=user)
token = self.model('token').createToken(user=user, days=3)
task = {
'mode': 'python',
'script': _climos_script,
'inputs': [{
'id': 'in_dir',
'type': 'string',
'format': 'text'
}, {
'id': 'out_filename',
'type': 'string',
'format': 'text'
}, {
'id': 'variables',
'type': 'python',
'format': 'object'
}, {
'id': 'seasons',
'type': 'python',
'format': 'object'
}],
'outputs': [{
'id': 'outfile',
'type': 'string',
'format': 'text'
}]
}
girderIoParams = {
'mode': 'girder',
'host': urlParts.hostname,
'port': urlParts.port,
'api_root': rest.getApiUrl(urlParts.path),
'scheme': urlParts.scheme,
'token': token['_id']
}
inputs = {
'in_dir': dict(girderIoParams, **{
'method': 'GET',
'id': str(inFolder['_id']),
'resource_type': 'folder',
'type': 'string',
'format': 'text',
'name': inFolder['name']
}),
'seasons': {
'mode': 'inline',
'type': 'python',
'format': 'object',
'data': json.loads(params['seasons'])
},
'variables': {
'mode': 'inline',
'type': 'python',
'format': 'object',
'data': json.loads(params['vars'])
},
'out_filename': {
'mode': 'inline',
'type': 'string',
'format': 'text',
'data': params['outputFilename'].strip()
}
}
outputs = {
'outfile': dict(girderIoParams, **{
'parent_type': 'folder',
'parent_id': str(outFolder['_id']),
'format': 'text',
'type': 'string'
})
}
job['kwargs'] = {
'task': task,
'inputs': inputs,
'outputs': outputs,
'jobInfo': {
'method': 'PUT',
'url': '/'.join((apiUrl, 'job', str(job['_id']))),
'headers': {'Girder-Token': token['_id']},
'logPrint': True
},
'validate': False,
'auto_convert': True,
'cleanup': True
}
job = jobModel.save(job)
jobModel.scheduleJob(job)
return jobModel.filter(job, user)
def testGetApiUrl(self):
url = "https://localhost/thing/api/v1/hello/world?foo=bar#test"
self.assertEqual(rest.getApiUrl(url), "https://localhost/thing/api/v1")
url = "https://localhost/girder#users"
self.assertRaises(Exception, rest.getApiUrl, url=url)
