def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage, params): jobModel = Job() token = Token().createToken(days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser()) job = jobModel.createJob(title='Read docker task specs: %s' % image, type='folder.item_task_slicer_cli_description', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] jobOptions = { 'itemTaskId': folder['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'POST', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'folder', str(folder['_id']), 'item_task_slicer_cli_xml')), 'headers': { 'Girder-Token': token['_id'] }, 'params': { 'image': image, 'args': json.dumps(args), 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def runJsonTasksDescription(self, folder, image, pullImage, params): jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser()) job = jobModel.createJob(title='Read docker task specs: %s' % image, type='item_task.json_description', handler='worker_handler', user=self.getCurrentUser()) jobOptions = { 'itemTaskId': folder['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': [], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'POST', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), self.resourceName, str(folder['_id']), 'json_specs')), 'headers': { 'Girder-Token': token['_id'] }, 'params': { 'image': image, 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False, 'cleanup': True } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage, params): jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser()) job = jobModel.createJob( title='Read docker task specs: %s' % image, type='folder.item_task_slicer_cli_description', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] jobOptions = { 'itemTaskId': folder['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'POST', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'folder', str(folder['_id']), 'item_task_slicer_cli_xml')), 'headers': {'Girder-Token': token['_id']}, 'params': { 'image': image, 'args': json.dumps(args), 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def attach_girder_api_url(sender=None, body=None, exchange=None, routing_key=None, headers=None, properties=None, declare=None, retry_policy=None, **kwargs): try: # girder v2 worker plugin from girder.plugins.worker import utils except ImportError: # girder v3 worker plugin from girder_worker.girder_plugin import utils headers['girder_api_url'] = utils.getWorkerApiUrl()
def processImages(self, folder, params): """ Process the images of a directory. This includes computing descriptors as well as training ITQ and computing hash codes. :param folder: A folder to process images on. """ def oldestFileId(item): """ Find the oldest file in an item, and return its id. :param item: An item document, or minimally a dictionary with the item id. :returns: The id of the oldest file, or False if the item has no files. """ files = ModelImporter.model('item').childFiles( item, limit=1, sort=[('created', SortDir.ASCENDING)]) try: return (str(item['_id']), files[0]['_id']) except Exception: return (False, False) # TODO Use a more granular token. # Ideally this would be scoped to only allow: # - Job Updates # - Data management of folder # - Retrieval of SMQTK settings token = ModelImporter.model('token').createToken( user=getCurrentUser(), days=1, scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ)) # TODO Filter items by supported mime types for SMQTK items = itertools.ifilter( lambda item: 'smqtk_uuid' not in item.get('meta', {}), ModelImporter.model('folder').childItems(folder)) itemFileIds = itertools.ifilter(None, itertools.imap(oldestFileId, items)) dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'], getWorkerApiUrl(), fileId)) for (itemId, fileId) in itemFileIds] return process_images.delay(str(folder['_id']), dataElementUris, girder_job_title='Processing Images', girder_job_type='GPU', girder_client_token=str(token['_id']))
def girder_before_task_publish(sender=None, body=None, exchange=None, routing_key=None, headers=None, properties=None, declare=None, retry_policy=None, **kwargs): if 'jobInfoSpec' not in headers: try: # Note: If we can import these objects from the girder packages we # assume our producer is in a girder REST request. This allows # us to create the job model's directly. Otherwise there will be an # ImportError and we can create the job via a REST request using # the jobInfoSpec in headers. from girder.utility.model_importer import ModelImporter from girder.plugins.worker import utils from girder.api.rest import getCurrentUser job_model = ModelImporter.model('job', 'jobs') user = headers.pop('girder_user', getCurrentUser()) token = headers.pop('girder_token', None) task_args, task_kwargs = body[0], body[1] job = job_model.createJob( **{'title': headers.get('girder_job_title', Task._girder_job_title), 'type': headers.get('girder_job_type', Task._girder_job_type), 'handler': headers.get('girder_job_handler', Task._girder_job_handler), 'public': headers.get('girder_job_public', Task._girder_job_public), 'user': user, 'args': task_args, 'kwargs': task_kwargs, 'otherFields': dict(celeryTaskId=headers['id'], **headers.get('girder_job_other_fields', Task._girder_job_other_fields))}) # If we don't have a token from girder_token kwarg, use # the job token instead. Otherwise no token if token is None: token = job.get('token', None) headers['jobInfoSpec'] = utils.jobInfoSpec(job, token) headers['apiUrl'] = utils.getWorkerApiUrl() except ImportError: # TODO: Check for self.job_manager to see if we have # tokens etc to contact girder and create a job model # we may be in a chain or a chord or some-such pass
def _processImages(folder, itemFilePairs): """ Create and schedule a Girder job for processing images. :param folder: Folder to process images for. :param fileIds: File IDs to process, these are converted into Girder data elemnts. """ jobModel = ModelImporter.model('job', 'jobs') # TODO Use a more granular token. # Ideally this would be scoped to only allow: # - Job Updates # - Data management of folder # - Retrieval of SMQTK settings token = ModelImporter.model('token').createToken( user=getCurrentUser(), days=1, scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ)) dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'], getWorkerApiUrl(), fileId)) for (itemId, fileId) in itemFilePairs] job = jobModel.createJob(title='Processing Images', type='GPU', handler='worker_handler', user=getCurrentUser(), args=(str(folder['_id']), dataElementUris), otherFields={ 'celeryTaskName': 'smqtk_worker.tasks.process_images', 'celeryQueue': 'process-images' }) job['token'] = token logger.info('assigning token %s' % token['_id']) jobModel.save(job) jobModel.scheduleJob(job)
def _processImages(folder, itemFilePairs): """ Create and schedule a Girder job for processing images. :param folder: Folder to process images for. :param fileIds: File IDs to process, these are converted into Girder data elemnts. """ jobModel = ModelImporter.model('job', 'jobs') # TODO Use a more granular token. # Ideally this would be scoped to only allow: # - Job Updates # - Data management of folder # - Retrieval of SMQTK settings token = ModelImporter.model('token').createToken(user=getCurrentUser(), days=1, scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ)) dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'], getWorkerApiUrl(), fileId)) for (itemId, fileId) in itemFilePairs] job = jobModel.createJob(title='Processing Images', type='GPU', handler='worker_handler', user=getCurrentUser(), args=(str(folder['_id']), dataElementUris), otherFields={'celeryTaskName': 'smqtk_worker.tasks.process_images', 'celeryQueue': 'process-images'}) job['token'] = token logger.info('assigning token %s' % token['_id']) jobModel.save(job) jobModel.scheduleJob(job)
def runSlicerCliTasksDescriptionForItem(self, item, image, args, setName, setDescription, pullImage, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = Job() token = Token().createToken(days=3, scope='item_task.set_task_spec.%s' % item['_id']) job = jobModel.createJob(title='Read docker Slicer CLI: %s' % image, type='item.item_task_slicer_cli_description', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] job.update({ 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']), 'item_task_slicer_cli_xml')), 'params': { 'setName': setName, 'setDescription': setDescription }, 'headers': { 'Girder-Token': token['_id'] } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } }) item['meta']['itemTaskSpec'] = {'mode': 'docker', 'docker_image': image} if args: item['meta']['itemTaskSlicerCliArgs'] = args Item().save(item) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def runSlicerCliTasksDescriptionForItem( self, item, image, args, setName, setDescription, pullImage, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = Job() token = Token().createToken( days=3, scope='item_task.set_task_spec.%s' % item['_id']) job = jobModel.createJob( title='Read docker Slicer CLI: %s' % image, type='item.item_task_slicer_cli_description', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] job.update({ 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']), 'item_task_slicer_cli_xml')), 'params': { 'setName': setName, 'setDescription': setDescription }, 'headers': {'Girder-Token': token['_id']} } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } }) item['meta']['itemTaskSpec'] = { 'mode': 'docker', 'docker_image': image } if args: item['meta']['itemTaskSlicerCliArgs'] = args Item().save(item) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def attach_girder_api_url(sender=None, body=None, exchange=None, routing_key=None, headers=None, properties=None, declare=None, retry_policy=None, **kwargs): from girder.plugins.worker import utils headers['girder_api_url'] = utils.getWorkerApiUrl()
def girder_before_task_publish(sender=None, body=None, exchange=None, routing_key=None, headers=None, properties=None, declare=None, retry_policy=None, **kwargs): try: if 'jobInfoSpec' not in headers: try: # Note: If we can import these objects from the girder packages we # assume our producer is in a girder REST request. This allows # us to create the job model's directly. Otherwise there will be an # ImportError and we can create the job via a REST request using # the jobInfoSpec in headers. from girder.utility.model_importer import ModelImporter from girder.plugins.worker import utils from girder.api.rest import getCurrentUser job_model = ModelImporter.model('job', 'jobs') user = headers.pop('girder_user', getCurrentUser()) # Sanitize any Transform objects task_args = tuple(_walk_obj(body[0], _maybe_model_repr)) task_kwargs = _walk_obj(body[1], _maybe_model_repr) job = job_model.createJob( **{ 'title': headers.pop('girder_job_title', Task._girder_job_title), 'type': headers.pop('girder_job_type', Task._girder_job_type), 'handler': headers.pop('girder_job_handler', Task._girder_job_handler), 'public': headers.pop('girder_job_public', Task._girder_job_public), 'user': user, 'args': task_args, 'kwargs': task_kwargs, 'otherFields': dict(celeryTaskId=headers['id'], **headers.pop('girder_job_other_fields', Task._girder_job_other_fields)) }) headers['jobInfoSpec'] = utils.jobInfoSpec(job) except ImportError: # TODO: Check for self.job_manager to see if we have # tokens etc to contact girder and create a job model # we may be in a chain or a chord or some-such pass if 'girder_api_url' not in headers: try: from girder.plugins.worker import utils headers['girder_api_url'] = utils.getWorkerApiUrl() except ImportError: # TODO: handle situation where girder_worker is producing # the message Note - this may not come up at all # depending on how we pass girder_api_url through to # the next task (e.g. in the context of chaining # events) pass if 'girder_client_token' not in headers: try: from girder.utility.model_importer import ModelImporter headers['girder_client_token'] = \ ModelImporter.model('token').createToken() except ImportError: # TODO: handle situation where girder_worker is producing # the message Note - this may not come up at all # depending on how we pass girder_token through to # the next task (e.g. in the context of chaining # events) pass if 'girder_result_hooks' in headers: # Celery task headers are not automatically serialized by celery # before being passed off to ampq for byte packing. We will have # to do that here. p = jsonpickle.pickler.Pickler() headers['girder_result_hooks'] = \ [p.flatten(grh) for grh in headers['girder_result_hooks']] # Finally, remove all reserved_options from headers for key in Task.reserved_options: headers.pop(key, None) except Exception: logger.exception('An error occurred in girder_before_task_publish.') raise
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName, setDescription, pullImage, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % item['_id'], user=self.getCurrentUser()) job = jobModel.createJob(title='Read docker task specs: %s' % image, type='item.item_task_json_description', handler='worker_handler', user=self.getCurrentUser()) jobOptions = { 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': [], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']), 'item_task_json_specs')), 'headers': { 'Girder-Token': token['_id'] }, 'params': { 'image': image, 'taskName': taskName, 'setName': setName, 'setDescription': setDescription, 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName, setDescription, pullImage, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = self.model('job', 'jobs') token = self.model('token').createToken( days=3, scope='item_task.set_task_spec.%s' % item['_id'], user=self.getCurrentUser()) job = jobModel.createJob( title='Read docker task specs: %s' % image, type='item.item_task_json_description', handler='worker_handler', user=self.getCurrentUser()) jobOptions = { 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': [], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']), 'item_task_json_specs')), 'headers': {'Girder-Token': token['_id']}, 'params': { 'image': image, 'taskName': taskName, 'setName': setName, 'setDescription': setDescription, 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job