def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage,
params):
jobModel = Job()
token = Token().createToken(days=3,
scope='item_task.set_task_spec.%s' %
folder['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(title='Read docker task specs: %s' % image,
type='folder.item_task_slicer_cli_description',
handler='worker_handler',
user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
jobOptions = {
'itemTaskId': folder['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'POST',
'format':
'text',
'url':
'/'.join((utils.getWorkerApiUrl(), 'folder',
str(folder['_id']), 'item_task_slicer_cli_xml')),
'headers': {
'Girder-Token': token['_id']
},
'params': {
'image': image,
'args': json.dumps(args),
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def runJsonTasksDescription(self, folder, image, pullImage, params):
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3,
scope='item_task.set_task_spec.%s' % folder['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(title='Read docker task specs: %s' % image,
type='item_task.json_description',
handler='worker_handler',
user=self.getCurrentUser())
jobOptions = {
'itemTaskId': folder['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': [],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'POST',
'format':
'text',
'url':
'/'.join((utils.getWorkerApiUrl(), self.resourceName,
str(folder['_id']), 'json_specs')),
'headers': {
'Girder-Token': token['_id']
},
'params': {
'image': image,
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False,
'cleanup': True
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage, params):
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3, scope='item_task.set_task_spec.%s' % folder['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(
title='Read docker task specs: %s' % image, type='folder.item_task_slicer_cli_description',
handler='worker_handler', user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
jobOptions = {
'itemTaskId': folder['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode': 'http',
'method': 'POST',
'format': 'text',
'url': '/'.join((utils.getWorkerApiUrl(), 'folder', str(folder['_id']),
'item_task_slicer_cli_xml')),
'headers': {'Girder-Token': token['_id']},
'params': {
'image': image,
'args': json.dumps(args),
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def attach_girder_api_url(sender=None,
body=None,
exchange=None,
routing_key=None,
headers=None,
properties=None,
declare=None,
retry_policy=None,
**kwargs):
try:
# girder v2 worker plugin
from girder.plugins.worker import utils
except ImportError:
# girder v3 worker plugin
from girder_worker.girder_plugin import utils
headers['girder_api_url'] = utils.getWorkerApiUrl()
def processImages(self, folder, params):
"""
Process the images of a directory. This includes computing descriptors
as well as training ITQ and computing hash codes.
:param folder: A folder to process images on.
"""
def oldestFileId(item):
"""
Find the oldest file in an item, and return its id.
:param item: An item document, or minimally a dictionary with the item id.
:returns: The id of the oldest file, or False if the item has no files.
"""
files = ModelImporter.model('item').childFiles(
item, limit=1, sort=[('created', SortDir.ASCENDING)])
try:
return (str(item['_id']), files[0]['_id'])
except Exception:
return (False, False)
# TODO Use a more granular token.
# Ideally this would be scoped to only allow:
# - Job Updates
# - Data management of folder
# - Retrieval of SMQTK settings
token = ModelImporter.model('token').createToken(
user=getCurrentUser(),
days=1,
scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ))
# TODO Filter items by supported mime types for SMQTK
items = itertools.ifilter(
lambda item: 'smqtk_uuid' not in item.get('meta', {}),
ModelImporter.model('folder').childItems(folder))
itemFileIds = itertools.ifilter(None,
itertools.imap(oldestFileId, items))
dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' %
(token['_id'], getWorkerApiUrl(), fileId))
for (itemId, fileId) in itemFileIds]
return process_images.delay(str(folder['_id']),
dataElementUris,
girder_job_title='Processing Images',
girder_job_type='GPU',
girder_client_token=str(token['_id']))
def girder_before_task_publish(sender=None, body=None, exchange=None,
routing_key=None, headers=None, properties=None,
declare=None, retry_policy=None, **kwargs):
if 'jobInfoSpec' not in headers:
try:
# Note: If we can import these objects from the girder packages we
# assume our producer is in a girder REST request. This allows
# us to create the job model's directly. Otherwise there will be an
# ImportError and we can create the job via a REST request using
# the jobInfoSpec in headers.
from girder.utility.model_importer import ModelImporter
from girder.plugins.worker import utils
from girder.api.rest import getCurrentUser
job_model = ModelImporter.model('job', 'jobs')
user = headers.pop('girder_user', getCurrentUser())
token = headers.pop('girder_token', None)
task_args, task_kwargs = body[0], body[1]
job = job_model.createJob(
**{'title': headers.get('girder_job_title', Task._girder_job_title),
'type': headers.get('girder_job_type', Task._girder_job_type),
'handler': headers.get('girder_job_handler', Task._girder_job_handler),
'public': headers.get('girder_job_public', Task._girder_job_public),
'user': user,
'args': task_args,
'kwargs': task_kwargs,
'otherFields': dict(celeryTaskId=headers['id'],
**headers.get('girder_job_other_fields',
Task._girder_job_other_fields))})
# If we don't have a token from girder_token kwarg, use
# the job token instead. Otherwise no token
if token is None:
token = job.get('token', None)
headers['jobInfoSpec'] = utils.jobInfoSpec(job, token)
headers['apiUrl'] = utils.getWorkerApiUrl()
except ImportError:
# TODO: Check for self.job_manager to see if we have
# tokens etc to contact girder and create a job model
# we may be in a chain or a chord or some-such
pass
def _processImages(folder, itemFilePairs):
"""
Create and schedule a Girder job for processing images.
:param folder: Folder to process images for.
:param fileIds: File IDs to process, these are converted into Girder data elemnts.
"""
jobModel = ModelImporter.model('job', 'jobs')
# TODO Use a more granular token.
# Ideally this would be scoped to only allow:
# - Job Updates
# - Data management of folder
# - Retrieval of SMQTK settings
token = ModelImporter.model('token').createToken(
user=getCurrentUser(),
days=1,
scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ))
dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' %
(token['_id'], getWorkerApiUrl(), fileId))
for (itemId, fileId) in itemFilePairs]
job = jobModel.createJob(title='Processing Images',
type='GPU',
handler='worker_handler',
user=getCurrentUser(),
args=(str(folder['_id']), dataElementUris),
otherFields={
'celeryTaskName':
'smqtk_worker.tasks.process_images',
'celeryQueue': 'process-images'
})
job['token'] = token
logger.info('assigning token %s' % token['_id'])
jobModel.save(job)
jobModel.scheduleJob(job)
def _processImages(folder, itemFilePairs):
"""
Create and schedule a Girder job for processing images.
:param folder: Folder to process images for.
:param fileIds: File IDs to process, these are converted into Girder data elemnts.
"""
jobModel = ModelImporter.model('job', 'jobs')
# TODO Use a more granular token.
# Ideally this would be scoped to only allow:
# - Job Updates
# - Data management of folder
# - Retrieval of SMQTK settings
token = ModelImporter.model('token').createToken(user=getCurrentUser(),
days=1,
scope=(TokenScope.USER_AUTH,
SMQTK_SETTING_READ))
dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'],
getWorkerApiUrl(),
fileId))
for (itemId, fileId) in itemFilePairs]
job = jobModel.createJob(title='Processing Images',
type='GPU',
handler='worker_handler',
user=getCurrentUser(),
args=(str(folder['_id']), dataElementUris),
otherFields={'celeryTaskName': 'smqtk_worker.tasks.process_images',
'celeryQueue': 'process-images'})
job['token'] = token
logger.info('assigning token %s' % token['_id'])
jobModel.save(job)
jobModel.scheduleJob(job)
def runSlicerCliTasksDescriptionForItem(self, item, image, args, setName,
setDescription, pullImage, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec',
{}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = Job()
token = Token().createToken(days=3,
scope='item_task.set_task_spec.%s' %
item['_id'])
job = jobModel.createJob(title='Read docker Slicer CLI: %s' % image,
type='item.item_task_slicer_cli_description',
handler='worker_handler',
user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
job.update({
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'PUT',
'format':
'text',
'url':
'/'.join((utils.getWorkerApiUrl(), 'item',
str(item['_id']), 'item_task_slicer_cli_xml')),
'params': {
'setName': setName,
'setDescription': setDescription
},
'headers': {
'Girder-Token': token['_id']
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
})
item['meta']['itemTaskSpec'] = {'mode': 'docker', 'docker_image': image}
if args:
item['meta']['itemTaskSlicerCliArgs'] = args
Item().save(item)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def runSlicerCliTasksDescriptionForItem(
self, item, image, args, setName, setDescription, pullImage, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = Job()
token = Token().createToken(
days=3, scope='item_task.set_task_spec.%s' % item['_id'])
job = jobModel.createJob(
title='Read docker Slicer CLI: %s' % image, type='item.item_task_slicer_cli_description',
handler='worker_handler', user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
job.update({
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode': 'http',
'method': 'PUT',
'format': 'text',
'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']),
'item_task_slicer_cli_xml')),
'params': {
'setName': setName,
'setDescription': setDescription
},
'headers': {'Girder-Token': token['_id']}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
})
item['meta']['itemTaskSpec'] = {
'mode': 'docker',
'docker_image': image
}
if args:
item['meta']['itemTaskSlicerCliArgs'] = args
Item().save(item)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def attach_girder_api_url(sender=None, body=None, exchange=None,
routing_key=None, headers=None, properties=None,
declare=None, retry_policy=None, **kwargs):
from girder.plugins.worker import utils
headers['girder_api_url'] = utils.getWorkerApiUrl()
def girder_before_task_publish(sender=None,
body=None,
exchange=None,
routing_key=None,
headers=None,
properties=None,
declare=None,
retry_policy=None,
**kwargs):
try:
if 'jobInfoSpec' not in headers:
try:
# Note: If we can import these objects from the girder packages we
# assume our producer is in a girder REST request. This allows
# us to create the job model's directly. Otherwise there will be an
# ImportError and we can create the job via a REST request using
# the jobInfoSpec in headers.
from girder.utility.model_importer import ModelImporter
from girder.plugins.worker import utils
from girder.api.rest import getCurrentUser
job_model = ModelImporter.model('job', 'jobs')
user = headers.pop('girder_user', getCurrentUser())
# Sanitize any Transform objects
task_args = tuple(_walk_obj(body[0], _maybe_model_repr))
task_kwargs = _walk_obj(body[1], _maybe_model_repr)
job = job_model.createJob(
**{
'title':
headers.pop('girder_job_title',
Task._girder_job_title),
'type':
headers.pop('girder_job_type', Task._girder_job_type),
'handler':
headers.pop('girder_job_handler',
Task._girder_job_handler),
'public':
headers.pop('girder_job_public',
Task._girder_job_public),
'user':
user,
'args':
task_args,
'kwargs':
task_kwargs,
'otherFields':
dict(celeryTaskId=headers['id'],
**headers.pop('girder_job_other_fields',
Task._girder_job_other_fields))
})
headers['jobInfoSpec'] = utils.jobInfoSpec(job)
except ImportError:
# TODO: Check for self.job_manager to see if we have
# tokens etc to contact girder and create a job model
# we may be in a chain or a chord or some-such
pass
if 'girder_api_url' not in headers:
try:
from girder.plugins.worker import utils
headers['girder_api_url'] = utils.getWorkerApiUrl()
except ImportError:
# TODO: handle situation where girder_worker is producing
# the message Note - this may not come up at all
# depending on how we pass girder_api_url through to
# the next task (e.g. in the context of chaining
# events)
pass
if 'girder_client_token' not in headers:
try:
from girder.utility.model_importer import ModelImporter
headers['girder_client_token'] = \
ModelImporter.model('token').createToken()
except ImportError:
# TODO: handle situation where girder_worker is producing
# the message Note - this may not come up at all
# depending on how we pass girder_token through to
# the next task (e.g. in the context of chaining
# events)
pass
if 'girder_result_hooks' in headers:
# Celery task headers are not automatically serialized by celery
# before being passed off to ampq for byte packing. We will have
# to do that here.
p = jsonpickle.pickler.Pickler()
headers['girder_result_hooks'] = \
[p.flatten(grh) for grh in headers['girder_result_hooks']]
# Finally, remove all reserved_options from headers
for key in Task.reserved_options:
headers.pop(key, None)
except Exception:
logger.exception('An error occurred in girder_before_task_publish.')
raise
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName,
setDescription, pullImage, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec',
{}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3,
scope='item_task.set_task_spec.%s' % item['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(title='Read docker task specs: %s' % image,
type='item.item_task_json_description',
handler='worker_handler',
user=self.getCurrentUser())
jobOptions = {
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': [],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode':
'http',
'method':
'PUT',
'format':
'text',
'url':
'/'.join((utils.getWorkerApiUrl(), 'item',
str(item['_id']), 'item_task_json_specs')),
'headers': {
'Girder-Token': token['_id']
},
'params': {
'image': image,
'taskName': taskName,
'setName': setName,
'setDescription': setDescription,
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName, setDescription,
pullImage, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = self.model('job', 'jobs')
token = self.model('token').createToken(
days=3, scope='item_task.set_task_spec.%s' % item['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(
title='Read docker task specs: %s' % image, type='item.item_task_json_description',
handler='worker_handler', user=self.getCurrentUser())
jobOptions = {
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': [],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode': 'http',
'method': 'PUT',
'format': 'text',
'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']),
'item_task_json_specs')),
'headers': {'Girder-Token': token['_id']},
'params': {
'image': image,
'taskName': taskName,
'setName': setName,
'setDescription': setDescription,
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job