def testWorkerStatusEndpoint(server, models): # Create a job to be handled by the worker plugin job = Job().createJob( title='title', type='foo', handler='worker_handler', user=models['admin'], public=False, args=(), kwargs={}) job['kwargs'] = { 'jobInfo': utils.jobInfoSpec(job), 'inputs': [ utils.girderInputSpec(models['adminFolder'], resourceType='folder') ], 'outputs': [ utils.girderOutputSpec(models['adminFolder'], token=models['adminToken']) ] } job = Job().save(job) assert job['status'] == JobStatus.INACTIVE # Schedule the job with mock.patch('celery.Celery') as celeryMock: instance = celeryMock.return_value instance.send_task.return_value = FakeAsyncResult() Job().scheduleJob(job) # Call the worker status endpoint resp = server.request('/worker/status', method='GET', user=models['admin']) assertStatusOk(resp) for key in ['report', 'stats', 'ping', 'active', 'reserved']: assert key in resp.json
def load(self, info): getPlugin('jobs').load(info) info['apiRoot'].nli = NLI() events.bind('jobs.job.update.after', 'nlisim', update_status) job_model = Job() job_model.exposeFields(level=constants.AccessType.ADMIN, fields={'args', 'kwargs'})
def cache_tile_frames_job(job): from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job from girder_large_image.models.image_item import ImageItem from girder import logger kwargs = job['kwargs'] item = ImageItem().load(kwargs.pop('itemId'), force=True) job = Job().updateJob(job, log='Started caching tile frames\n', status=JobStatus.RUNNING) try: for entry in kwargs.get('tileFramesList'): job = Job().load(job['_id'], force=True) if job['status'] == JobStatus.CANCELED: return job = Job().updateJob(job, log='Caching %r\n' % entry) ImageItem().tileFrames(item, checkAndCreate=True, **entry) job = Job().updateJob(job, log='Finished caching tile frames\n', status=JobStatus.SUCCESS) except Exception as exc: logger.exception('Failed caching tile frames') job = Job().updateJob(job, log='Failed caching tile frames (%s)\n' % exc, status=JobStatus.ERROR)
def schedule(event): """ This is bound to the "jobs.schedule" event, and will be triggered any time a job is scheduled. This handler will process any job that has the handler field set to "worker_handler". """ job = event.info if job['handler'] == 'worker_handler': task = job.get('celeryTaskName', 'girder_worker.run') # Set the job status to queued Job().updateJob(job, status=JobStatus.QUEUED) # Send the task to celery asyncResult = getCeleryApp().send_task(task, job['args'], job['kwargs'], queue=job.get('celeryQueue'), headers={ 'jobInfoSpec': jobInfoSpec( job, job.get('token', None)), 'apiUrl': getWorkerApiUrl() }) # Record the task ID from celery. Job().updateJob(job, otherFields={'celeryTaskId': asyncResult.task_id}) # Stop event propagation since we have taken care of scheduling. event.stopPropagation()
def scheduleThumbnailJob(file, attachToType, attachToId, user, width=0, height=0, crop=True): """ Schedule a local thumbnail creation job and return it. """ job = Job().createLocalJob(title='Generate thumbnail for %s' % file['name'], user=user, type='thumbnails.create', public=False, module='girder_thumbnails.worker', kwargs={ 'fileId': str(file['_id']), 'width': width, 'height': height, 'crop': crop, 'attachToType': attachToType, 'attachToId': str(attachToId) }) Job().scheduleJob(job) return job
def createThumbnails(self, params): self.requireParams(['spec'], params) try: spec = json.loads(params['spec']) if not isinstance(spec, list): raise ValueError() except ValueError: raise RestException('The spec parameter must be a JSON list.') maxThumbnailFiles = int(Setting().get( constants.PluginSettings.LARGE_IMAGE_MAX_THUMBNAIL_FILES)) if maxThumbnailFiles <= 0: raise RestException('Thumbnail files are not enabled.') jobKwargs = {'spec': spec} if params.get('logInterval') is not None: jobKwargs['logInterval'] = float(params['logInterval']) if params.get('concurrent') is not None: jobKwargs['concurrent'] = float(params['concurrent']) job = Job().createLocalJob( module='girder_large_image.rest.large_image_resource', function='createThumbnailsJob', kwargs=jobKwargs, title='Create large image thumbnail files.', type='large_image_create_thumbnails', user=self.getCurrentUser(), public=True, asynchronous=True, ) Job().scheduleJob(job) return job
def _onUpload(event): """ Look at uploads containing references related to this plugin. If found, they are used to link item task outputs back to a job document. """ try: ref = json.loads(event.info.get('reference', '')) except ValueError: return if isinstance(ref, dict) and ref.get('type') == 'item_tasks.output': jobModel = Job() tokenModel = Token() token = event.info['currentToken'] if tokenModel.hasScope(token, 'item_tasks.job_write:%s' % ref['jobId']): job = jobModel.load(ref['jobId'], force=True, exc=True) else: job = jobModel.load( ref['jobId'], level=AccessType.WRITE, user=event.info['currentUser'], exc=True) file = event.info['file'] item = Item().load(file['itemId'], force=True) # Add link to job model to the output item jobModel.updateJob(job, otherFields={ 'itemTaskBindings.outputs.%s.itemId' % ref['id']: item['_id'] }) # Also a link in the item to the job that created it item['createdByJob'] = job['_id'] Item().save(item)
def convertImage(self, item, fileObj, user=None, token=None, localJob=True, **kwargs): if fileObj['itemId'] != item['_id']: raise TileGeneralException( 'The provided file must be in the provided item.') if not localJob: return self._convertImageViaWorker(item, fileObj, user, token, **kwargs) # local job job = Job().createLocalJob( module='large_image_tasks.tasks', function='convert_image_job', kwargs={ 'itemId': str(item['_id']), 'fileId': str(fileObj['_id']), 'userId': str(user['_id']) if user else None, **kwargs, }, title='Convert a file to a large image file.', type='large_image_convert_image', user=user, public=True, asynchronous=True, ) Job().scheduleJob(job) return job
def setUp(self): base.TestCase.setUp(self) self.users = [User().createUser( 'usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n) for n in range(3)] self.jobModel = Job()
def testDeleteIncompleteTile(server, admin, user, fsAssetstore, unavailableWorker): # noqa # Test the large_image/settings end point resp = server.request(method='DELETE', path='/large_image/tiles/incomplete', user=user) assert utilities.respStatus(resp) == 403 resp = server.request(method='DELETE', path='/large_image/tiles/incomplete', user=admin) assert utilities.respStatus(resp) == 200 results = resp.json assert results['removed'] == 0 file = utilities.uploadTestFile('yb10kx5k.png', admin, fsAssetstore) itemId = str(file['itemId']) resp = server.request(method='POST', path='/item/%s/tiles' % itemId, user=admin) resp = server.request(method='DELETE', path='/large_image/tiles/incomplete', user=admin) assert utilities.respStatus(resp) == 200 results = resp.json assert results['removed'] == 1 def preventCancel(evt): job = evt.info['job'] params = evt.info['params'] if (params.get('status') and params.get('status') != job['status'] and params['status'] in (JobStatus.CANCELED, CustomJobStatus.CANCELING)): evt.preventDefault() # Prevent a job from cancelling events.bind('jobs.job.update', 'testDeleteIncompleteTile', preventCancel) # Create a job and mark it as running resp = server.request(method='POST', path='/item/%s/tiles' % itemId, user=admin) job = Job().load(id=resp.json['_id'], force=True) Job().updateJob(job, status=JobStatus.RUNNING) resp = server.request(method='DELETE', path='/large_image/tiles/incomplete', user=admin) events.unbind('jobs.job.update', 'testDeleteIncompleteTile') assert utilities.respStatus(resp) == 200 results = resp.json assert results['removed'] == 0 assert 'could not be canceled' in results['message'] # Now we should be able to cancel the job resp = server.request(method='DELETE', path='/large_image/tiles/incomplete', user=admin) assert utilities.respStatus(resp) == 200 results = resp.json assert results['removed'] == 1
def setUp(self): super().setUp() self.users = [ User().createUser('usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n) for n in range(3) ] self.jobModel = Job()
def convert_image_job(job): import tempfile from girder.constants import AccessType from girder.models.file import File from girder.models.folder import Folder from girder.models.item import Item from girder.models.upload import Upload from girder.models.user import User from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job kwargs = job['kwargs'] item = Item().load(kwargs.pop('itemId'), force=True) fileObj = File().load(kwargs.pop('fileId'), force=True) userId = kwargs.pop('userId', None) user = User().load(userId, force=True) if userId else None folder = Folder().load(kwargs.pop('folderId', item['folderId']), user=user, level=AccessType.WRITE) name = kwargs.pop('name', None) job = Job().updateJob( job, log='Started large image conversion\n', status=JobStatus.RUNNING) logger = logging.getLogger('large-image-converter') handler = JobLogger(job=job) logger.addHandler(handler) # We could increase the default logging level here # logger.setLevel(logging.DEBUG) try: with tempfile.TemporaryDirectory() as tempdir: dest = create_tiff( inputFile=File().getLocalFilePath(fileObj), inputName=fileObj['name'], outputDir=tempdir, **kwargs, ) job = Job().updateJob(job, log='Storing result\n') with open(dest, 'rb') as fobj: Upload().uploadFromFile( fobj, size=os.path.getsize(dest), name=name or os.path.basename(dest), parentType='folder', parent=folder, user=user, ) except Exception as exc: status = JobStatus.ERROR logger.exception('Failed in large image conversion') job = Job().updateJob( job, log='Failed in large image conversion (%s)\n' % exc, status=status) else: status = JobStatus.SUCCESS job = Job().updateJob( job, log='Finished large image conversion\n', status=status) finally: logger.removeHandler(handler)
def testLocalJob(models): # Make sure local jobs still work job = Job().createLocalJob( title='local', type='local', user=models['admin'], module='plugin_tests.worker_test', function='local_job') Job().scheduleJob(job) job = Job().load(job['_id'], force=True, includeLog=True) assert 'job ran' in job['log']
def testWorkerCancel(models): jobModel = Job() job = jobModel.createJob( title='title', type='foo', handler='worker_handler', user=models['admin'], public=False, args=(), kwargs={}) job['kwargs'] = { 'jobInfo': utils.jobInfoSpec(job), 'inputs': [ utils.girderInputSpec(models['adminFolder'], resourceType='folder') ], 'outputs': [ utils.girderOutputSpec(models['adminFolder'], token=models['adminToken']) ] } job = jobModel.save(job) assert job['status'] == JobStatus.INACTIVE # Schedule the job, make sure it is sent to celery with mock.patch('celery.Celery') as celeryMock, \ mock.patch('girder_worker.girder_plugin.event_handlers.AsyncResult') as asyncResult: instance = celeryMock.return_value instance.send_task.return_value = FakeAsyncResult() jobModel.scheduleJob(job) jobModel.cancelJob(job) asyncResult.assert_called_with('fake_id', app=mock.ANY) # Check we called revoke asyncResult.return_value.revoke.assert_called_once() job = jobModel.load(job['_id'], force=True) assert job['status'] == CustomJobStatus.CANCELING
def execute_simulation(self, name, config, folder=None): target_time = config.get('simulation', {}).get('run_time', 50) user, token = self.getCurrentUser(returnToken=True) folder_model = Folder() job_model = Job() if folder is None: folder = folder_model.findOne( {'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user'} ) if folder is None: raise RestException('Could not find the user\'s "public" folder.') simulation_model = Simulation() simulation = simulation_model.createSimulation( folder, name, config, user, True, ) girder_config = GirderConfig( api=GIRDER_API, token=str(token['_id']), folder=str(folder['_id']) ) simulation_config = SimulationConfig(NLI_CONFIG_FILE, config) # TODO: This would be better stored as a dict, but it's easier once we change the # config object format. simulation_config_file = StringIO() simulation_config.write(simulation_config_file) job = job_model.createJob( title='NLI Simulation', type=NLI_JOB_TYPE, kwargs={ 'girder_config': attr.asdict(girder_config), 'simulation_config': simulation_config_file.getvalue(), 'config': config, 'simulation_id': simulation['_id'], }, user=user, ) simulation['nli']['job_id'] = job['_id'] simulation_model.save(simulation) run_simulation.delay( name=name, girder_config=girder_config, simulation_config=simulation_config, target_time=target_time, job=job, simulation_id=simulation['_id'], ) return job
def testWorkerWithParent(models): jobModel = Job() parentJob = jobModel.createJob( title='title', type='foo', handler='worker_handler', user=models['admin'], public=False, otherFields={'celeryTaskId': '1234'}) childJob = jobModel.createJob( title='title', type='foo', handler='worker_handler', user=models['admin'], public=False, otherFields={'celeryTaskId': '5678', 'celeryParentTaskId': '1234'}) assert parentJob['_id'] == childJob['parentId']
def import_recursive(job): try: root = job['kwargs']['root'] token = job['kwargs']['token'] user = User().load(job['userId'], force=True) children = list(Folder().childFolders(root, 'collection', user=user)) count = len(children) progress = 0 job = Job().updateJob(job, log='Started TCGA import\n', status=JobStatus.RUNNING, progressCurrent=progress, progressTotal=count) logger.info('Starting recursive TCGA import') for child in children: progress += 1 try: msg = 'Importing "%s"' % child.get('name', '') job = Job().updateJob(job, log=msg, progressMessage=msg + '\n', progressCurrent=progress) logger.debug(msg) Cohort().importDocument(child, recurse=True, user=user, token=token, job=job) job = Job().load(id=job['_id'], force=True) # handle any request to stop execution if (not job or job['status'] in (JobStatus.CANCELED, JobStatus.ERROR)): logger.info('TCGA import job halted with') return except ValidationException: logger.warning('Failed to import %s' % child.get('name', '')) logger.info('Starting recursive TCGA import') job = Job().updateJob(job, log='Finished TCGA import\n', status=JobStatus.SUCCESS, progressCurrent=count, progressMessage='Finished TCGA import') except Exception as e: logger.exception('Importing TCGA failed with %s' % str(e)) job = Job().updateJob(job, log='Import failed with %s\n' % str(e), status=JobStatus.ERROR)
def list_simulation_jobs(self, limit, offset, sort): user = self.getCurrentUser() job_model = Job() return job_model.list( types=[NLI_JOB_TYPE], statuses=[JobStatus.QUEUED, JobStatus.RUNNING], user=user, currentUser=user, limit=limit, offset=offset, sort=sort, )
def run_pipeline_task(self, folder, pipeline: PipelineDescription): """ Run a pipeline on a dataset. :param folder: The girder folder containing the dataset to run on. :param pipeline: The pipeline to run the dataset on. """ folder_id_str = str(folder["_id"]) # First, verify that no other outstanding jobs are running on this dataset existing_jobs = Job().findOne({ JOBCONST_DATASET_ID: folder_id_str, 'status': { # Find jobs that are inactive, queued, or running # https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py '$in': [0, 1, 2] }, }) if existing_jobs is not None: raise RestException( (f"A pipeline for {folder_id_str} is already running. " "Only one outstanding job may be run at a time for " "a dataset.")) user = self.getCurrentUser() token = Token().createToken(user=user, days=14) move_existing_result_to_auxiliary_folder(folder, user) params: PipelineJob = { "input_folder": folder_id_str, "input_type": folder["meta"]["type"], "output_folder": folder_id_str, "pipeline": pipeline, } newjob = run_pipeline.apply_async( queue="pipelines", kwargs=dict( params=params, girder_job_title= f"Running {pipeline['name']} on {str(folder['name'])}", girder_client_token=str(token["_id"]), girder_job_type="pipelines", ), ) newjob.job[JOBCONST_DATASET_ID] = folder_id_str newjob.job[JOBCONST_RESULTS_FOLDER_ID] = folder_id_str newjob.job[JOBCONST_PIPELINE_NAME] = pipeline['name'] # Allow any users with accecss to the input data to also # see and possibly manage the job Job().copyAccessPolicies(folder, newjob.job) Job().save(newjob.job) return newjob.job
def updateStep(self, slurmJobId): job = Job().findOne( {'otherFields.slurm_info.slurm_id': int(slurmJobId)}) # send log to girder periodic log_file_name = 'slurm-{}.{}.out'.format( job['otherFields']['slurm_info']['name'], slurmJobId) log_file_path = os.path.join(self._shared_partition_log, log_file_name) f = open(log_file_path, "r") content = f.read() job['log'].append(content) f.close() job.save() return 'Updated log'
def _handleUpload(event): upload, file = event.info['upload'], event.info['file'] try: reference = json.loads(upload.get('reference')) except (TypeError, ValueError): return if 'photomorph' in reference and 'photomorphOrdinal' in reference: # TODO this is insecure. Should access check the item. item = Item().load(file['itemId'], force=True, exc=True) item['originalName'] = item['name'] name = '%05d_%s' % (reference['photomorphOrdinal'], item['name']) item['name'] = name item['photomorphTakenDate'] = _extractDate(file) Item().save(item) file['name'] = name File().save(file) try: createThumbnail(width=128, height=128, crop=True, fileId=file['_id'], attachToType='item', attachToId=item['_id']) except Exception: logger.exception('Failure during photomorph thumbnailing') elif reference.get('photomorph') and 'resultType' in reference: Folder().update({'_id': ObjectId(reference['folderId'])}, { '$push': { 'photomorphOutputItems.%s' % reference['resultType']: { 'fileId': file['_id'], 'name': file['name'] } } }, multi=False) elif reference.get('inpaintedImage'): folder = Folder().load(reference['folderId'], user=getCurrentUser(), level=AccessType.WRITE) if 'inpaintingJobId' in folder: job = Job().load(folder['inpaintingJobId'], force=True) job['inpaintingImageResultId'] = file['_id'] Job().save(job)
def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage, params): jobModel = Job() token = Token().createToken( days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser()) job = jobModel.createJob( title='Read docker task specs: %s' % image, type='folder.item_task_slicer_cli_description', handler='worker_handler', user=self.getCurrentUser()) if args[-1:] == ['--xml']: args = args[:-1] jobOptions = { 'itemTaskId': folder['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': args + ['--xml'], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'POST', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'folder', str(folder['_id']), 'item_task_slicer_cli_xml')), 'headers': {'Girder-Token': token['_id']}, 'params': { 'image': image, 'args': json.dumps(args), 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
def executeTask(self, item, jobTitle, includeJobInfo, inputs, outputs): user = self.getCurrentUser() if jobTitle is None: jobTitle = item['name'] task, handler = self._validateTask(item) if task.get('mode') == 'girder_worker': return runCeleryTask(item['meta']['itemTaskImport'], inputs) jobModel = Job() job = jobModel.createJob( title=jobTitle, type='item_task', handler=handler, user=user) # If this is a user auth token, we make an IO-enabled token token = self.getCurrentToken() tokenModel = Token() if tokenModel.hasScope(token, TokenScope.USER_AUTH): token = tokenModel.createToken( user=user, days=7, scope=(TokenScope.DATA_READ, TokenScope.DATA_WRITE)) job['itemTaskTempToken'] = token['_id'] token = tokenModel.addScope(token, 'item_tasks.job_write:%s' % job['_id']) job.update({ 'itemTaskId': item['_id'], 'itemTaskBindings': { 'inputs': inputs, 'outputs': outputs }, 'kwargs': { 'task': task, 'inputs': self._transformInputs(inputs, token), 'outputs': self._transformOutputs(outputs, token, job, task, item['_id']), 'validate': False, 'auto_convert': False, 'cleanup': True } }) if includeJobInfo: job['kwargs']['jobInfo'] = utils.jobInfoSpec(job) if 'itemTaskCeleryQueue' in item.get('meta', {}): job['celeryQueue'] = item['meta']['itemTaskCeleryQueue'] job = jobModel.save(job) jobModel.scheduleJob(job) return job
def delete(self, item, skipFileIds=None): deleted = False if 'largeImage' in item: job = None if 'jobId' in item['largeImage']: try: job = Job().load(item['largeImage']['jobId'], force=True, exc=True) except ValidationException: # The job has been deleted, but we still need to clean up # the rest of the tile information pass if (item['largeImage'].get('expected') and job and job.get('status') in (JobStatus.QUEUED, JobStatus.RUNNING)): # cannot cleanly remove the large image, since a conversion # job is currently in progress # TODO: cancel the job # TODO: return a failure error code return False # If this file was created by the worker job, delete it if 'jobId' in item['largeImage']: # To eliminate all traces of the job, add # if job: # Job().remove(job) del item['largeImage']['jobId'] if 'originalId' in item['largeImage']: # The large image file should not be the original file assert item['largeImage']['originalId'] != \ item['largeImage'].get('fileId') if ('fileId' in item['largeImage'] and (not skipFileIds or item['largeImage']['fileId'] not in skipFileIds)): file = File().load(id=item['largeImage']['fileId'], force=True) if file: File().remove(file) del item['largeImage']['originalId'] del item['largeImage'] item = self.save(item) deleted = True self.removeThumbnailFiles(item) return deleted
def listInpaintingJobs(self, user, limit, offset, sort): user = user or self.getCurrentUser() cursor = Job().find( { 'userId': user['_id'], 'inpaintingImageId': { '$exists': True } }, sort=sort) return list(Job().filterResultsByPermission(cursor=cursor, user=self.getCurrentUser(), level=AccessType.READ, limit=limit, offset=offset))
def wait_for_status(user, job, status): """ Utility to wait for a job model to move into a particular state. :param job: The job model to wait on :param status: The state to wait for. :returns: True if the job model moved into the requested state, False otherwise. """ retries = 0 jobModel = Job() while retries < 10: job = jobModel.load(job['_id'], user=user) if job['status'] == status: return True return False
def runInpainting(self, image, mask, folder): basename = os.path.splitext(image['name'])[0] outPath = VolumePath(basename + '_result.jpg') artifactPath = VolumePath('job_artifacts') job = docker_run.delay( 'zachmullen/inpainting:latest', container_args=[ GirderFileIdToVolume(image['_id']), GirderFileIdToVolume(mask['_id']), outPath, '--artifacts-dir', artifactPath, '--progress-pipe', ProgressPipe() ], girder_job_title='Inpainting: %s' % image['name'], girder_result_hooks=[ GirderUploadVolumePathToFolder(outPath, folder['_id'], upload_kwargs={ 'reference': json.dumps({ 'inpaintedImage': True, 'folderId': str(folder['_id']), }) }), # GirderUploadVolumePathJobArtifact(artifactPath) ]).job folder['inpaintingJobId'] = job['_id'] Folder().save(folder) job['inpaintingImageId'] = image['_id'] job['inpaintingMaskId'] = mask['_id'] job['inpaintingFolderId'] = folder['_id'] return Job().save(job)
def attachParentJob(event): """Attach parentJob before a model is saved.""" job = event.info if job.get('celeryParentTaskId'): celeryParentTaskId = job['celeryParentTaskId'] parentJob = Job().findOne({'celeryTaskId': celeryParentTaskId}) event.info['parentId'] = parentJob['_id']
def jobInfoSpec(job, token=None, logPrint=True): """ Build the jobInfo specification for a task to write status and log output back to a Girder job. :param job: The job document representing the worker task. :type job: dict :param token: The token to use. Creates a job token if not passed. :type token: str or dict :param logPrint: Whether standard output from the job should be """ if token is None: token = Job().createJobToken(job) if isinstance(token, dict): token = token['_id'] return { 'method': 'PUT', 'url': '/'.join((getWorkerApiUrl(), 'job', str(job['_id']))), 'reference': str(job['_id']), 'headers': { 'Girder-Token': token }, 'logPrint': logPrint }
def execute_simulation(self, name, config, folder=None): target_time = config.get('simulation', {}).get('run_time', 96) user, token = self.getCurrentUser(returnToken=True) folder_model: Folder = Folder() job_model: Job = Job() if folder is None: folder = folder_model.findOne({ 'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user' }) if folder is None: raise RestException( 'Could not find the user\'s "public" folder.') job, simulation = simulation_runner( config=config, parent_folder=folder, job_model=job_model, run_name=name, target_time=target_time, token=token, user=user, ) return job
def _createThumbnails(server, admin, spec, cancel=False): params = {'spec': json.dumps(spec)} if cancel: params['logInterval'] = 0 resp = server.request(method='PUT', path='/large_image/thumbnails', user=admin, params=params) assert utilities.respStatus(resp) == 200 job = resp.json if cancel: job = _waitForJobToBeRunning(job) job = Job().cancelJob(job) starttime = time.time() while True: assert time.time() - starttime < 30 resp = server.request('/job/%s' % str(job['_id']), user=admin) assert utilities.respStatus(resp) == 200 if resp.json.get('status') == JobStatus.SUCCESS: return True if resp.json.get('status') == JobStatus.ERROR: return False if resp.json.get('status') == JobStatus.CANCELED: return 'canceled' time.sleep(0.1)
def cancel(event): """ This is bound to the "jobs.cancel" event, and will be triggered any time a job is canceled. This handler will process any job that has the handler field set to "worker_handler". """ job = event.info if job['handler'] in ['worker_handler', 'celery_handler']: # Stop event propagation and prevent default, we are using a custom state event.stopPropagation().preventDefault() celeryTaskId = job.get('celeryTaskId') if celeryTaskId is None: msg = ( "Unable to cancel Celery task. Job '%s' doesn't have a Celery task id." % job['_id']) logger.warn(msg) return if job['status'] not in [ CustomJobStatus.CANCELING, JobStatus.CANCELED, JobStatus.SUCCESS, JobStatus.ERROR ]: # Set the job status to canceling Job().updateJob(job, status=CustomJobStatus.CANCELING) # Send the revoke request. asyncResult = AsyncResult(celeryTaskId, app=getCeleryApp()) asyncResult.revoke()
def importCollection(self, params): user = self.getCurrentUser() token = self.getCurrentToken() tcga = self.getTCGACollection(level=AccessType.WRITE) job = Job().createLocalJob(module='histomicsui.rest.tcga', function='import_recursive', kwargs={ 'root': tcga, 'token': token }, title='Import TCGA items', user=user, type='tcga_import_recursive', public=False, asynchronous=True) Job().scheduleJob(job) return job
def run(job): jobModel = Job() jobModel.updateJob(job, status=JobStatus.RUNNING) try: newFile = createThumbnail(**job['kwargs']) log = 'Created thumbnail file %s.' % newFile['_id'] jobModel.updateJob(job, status=JobStatus.SUCCESS, log=log) except Exception: t, val, tb = sys.exc_info() log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb)) jobModel.updateJob(job, status=JobStatus.ERROR, log=log) raise
def onJobUpdate(event): """ Hook into job update event so we can look for job failure events and email the user and challenge/phase administrators accordingly. Here, an administrator is defined to be a user with WRITE access or above. """ isErrorStatus = False try: isErrorStatus = int(event.info['params'].get('status')) == JobStatus.ERROR except (ValueError, TypeError): pass if (event.info['job']['type'] == 'covalic_score' and isErrorStatus): covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix()) # Create minimal log that contains only Covalic errors. # Use full log if no Covalic-specific errors are found. # Fetch log from model, because log in event may not be up-to-date. job = Job().load( event.info['job']['_id'], includeLog=True, force=True) log = job.get('log') minimalLog = None if log: log = ''.join(log) minimalLog = '\n'.join([line[len(JOB_LOG_PREFIX):].strip() for line in log.splitlines() if line.startswith(JOB_LOG_PREFIX)]) if not minimalLog: minimalLog = log submission = Submission().load( event.info['job']['covalicSubmissionId']) phase = Phase().load( submission['phaseId'], force=True) challenge = Challenge().load( phase['challengeId'], force=True) user = User().load( event.info['job']['userId'], force=True) rescoring = job.get('rescoring', False) # Mail admins, include full log emails = sorted(getPhaseUserEmails( phase, AccessType.WRITE, includeChallengeUsers=True)) html = mail_utils.renderTemplate('covalic.submissionErrorAdmin.mako', { 'submission': submission, 'challenge': challenge, 'phase': phase, 'user': user, 'host': covalicHost, 'log': log }) mail_utils.sendEmail( to=emails, subject='Submission processing error', text=html) # Mail user, include minimal log if not rescoring: html = mail_utils.renderTemplate('covalic.submissionErrorUser.mako', { 'submission': submission, 'challenge': challenge, 'phase': phase, 'host': covalicHost, 'log': minimalLog }) mail_utils.sendEmail( to=user['email'], subject='Submission processing error', text=html)
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName, setDescription, pullImage, params): if 'meta' not in item: item['meta'] = {} if image is None: image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image') if not image: raise RestException( 'You must pass an image parameter, or set the itemTaskSpec.docker_image ' 'field of the item.') jobModel = Job() token = Token().createToken( days=3, scope='item_task.set_task_spec.%s' % item['_id'], user=self.getCurrentUser()) job = jobModel.createJob( title='Read docker task specs: %s' % image, type='item.item_task_json_description', handler='worker_handler', user=self.getCurrentUser()) jobOptions = { 'itemTaskId': item['_id'], 'kwargs': { 'task': { 'mode': 'docker', 'docker_image': image, 'container_args': [], 'pull_image': pullImage, 'outputs': [{ 'id': '_stdout', 'format': 'text' }], }, 'outputs': { '_stdout': { 'mode': 'http', 'method': 'PUT', 'format': 'text', 'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']), 'item_task_json_specs')), 'headers': {'Girder-Token': token['_id']}, 'params': { 'image': image, 'taskName': taskName, 'setName': setName, 'setDescription': setDescription, 'pullImage': pullImage } } }, 'jobInfo': utils.jobInfoSpec(job), 'validate': False, 'auto_convert': False } } job.update(jobOptions) job = jobModel.save(job) jobModel.scheduleJob(job) return job
class JobsTestCase(base.TestCase): def setUp(self): base.TestCase.setUp(self) self.users = [User().createUser( 'usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n) for n in range(3)] self.jobModel = Job() def testJobs(self): self.job = None def schedule(event): self.job = event.info if self.job['handler'] == 'my_handler': self.job['status'] = JobStatus.RUNNING self.job = self.jobModel.save(self.job) self.assertEqual(self.job['args'], ('hello', 'world')) self.assertEqual(self.job['kwargs'], {'a': 'b'}) events.bind('jobs.schedule', 'test', schedule) # Create a job job = self.jobModel.createJob( title='Job Title', type='my_type', args=('hello', 'world'), kwargs={'a': 'b'}, user=self.users[1], handler='my_handler', public=False) self.assertEqual(self.job, None) self.assertEqual(job['status'], JobStatus.INACTIVE) # Schedule the job, make sure our handler was invoked self.jobModel.scheduleJob(job) self.assertEqual(self.job['_id'], job['_id']) self.assertEqual(self.job['status'], JobStatus.RUNNING) # Since the job is not public, user 2 should not have access path = '/job/%s' % job['_id'] resp = self.request(path, user=self.users[2]) self.assertStatus(resp, 403) resp = self.request(path, user=self.users[2], method='PUT') self.assertStatus(resp, 403) resp = self.request(path, user=self.users[2], method='DELETE') self.assertStatus(resp, 403) # If no user is specified, we should get a 401 error resp = self.request(path, user=None) self.assertStatus(resp, 401) # Make sure user who created the job can see it resp = self.request(path, user=self.users[1]) self.assertStatusOk(resp) # We should be able to update the job as the user who created it resp = self.request(path, method='PUT', user=self.users[1], params={ 'log': 'My log message\n' }) self.assertStatusOk(resp) # We should be able to create a job token and use that to update it too token = self.jobModel.createJobToken(job) resp = self.request(path, method='PUT', params={ 'log': 'append message', 'token': token['_id'] }) self.assertStatusOk(resp) # We shouldn't get the log back in this case self.assertNotIn('log', resp.json) # Do a fetch on the job itself to get the log resp = self.request(path, user=self.users[1]) self.assertStatusOk(resp) self.assertEqual( resp.json['log'], ['My log message\n', 'append message']) # Test overwriting the log and updating status resp = self.request(path, method='PUT', params={ 'log': 'overwritten log', 'overwrite': 'true', 'status': JobStatus.SUCCESS, 'token': token['_id'] }) self.assertStatusOk(resp) self.assertNotIn('log', resp.json) self.assertEqual(resp.json['status'], JobStatus.SUCCESS) job = self.jobModel.load(job['_id'], force=True, includeLog=True) self.assertEqual(job['log'], ['overwritten log']) # We should be able to delete the job as the user who created it resp = self.request(path, user=self.users[1], method='DELETE') self.assertStatusOk(resp) job = self.jobModel.load(job['_id'], force=True) self.assertIsNone(job) def testLegacyLogBehavior(self): # Force save a job with a string log to simulate a legacy job record job = self.jobModel.createJob( title='legacy', type='legacy', user=self.users[1], save=False) job['log'] = 'legacy log' job = self.jobModel.save(job, validate=False) self.assertEqual(job['log'], 'legacy log') # Load the record, we should now get the log as a list job = self.jobModel.load(job['_id'], force=True, includeLog=True) self.assertEqual(job['log'], ['legacy log']) def testListJobs(self): job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=False) anonJob = self.jobModel.createJob(title='Anon job', type='t') # Ensure timestamp for public job is strictly higher (ms resolution) time.sleep(0.1) publicJob = self.jobModel.createJob( title='Anon job', type='t', public=True) # User 1 should be able to see their own jobs resp = self.request('/job', user=self.users[1], params={ 'userId': self.users[1]['_id'] }) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]['_id'], str(job['_id'])) # User 2 should not see user 1's jobs in the list resp = self.request('/job', user=self.users[2], params={ 'userId': self.users[1]['_id'] }) self.assertEqual(resp.json, []) # Omitting a userId should assume current user resp = self.request('/job', user=self.users[1]) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]['_id'], str(job['_id'])) # Explicitly passing "None" should show anonymous jobs resp = self.request('/job', user=self.users[0], params={ 'userId': 'none' }) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 2) self.assertEqual(resp.json[0]['_id'], str(publicJob['_id'])) self.assertEqual(resp.json[1]['_id'], str(anonJob['_id'])) # Non-admins should only see public anon jobs resp = self.request('/job', params={'userId': 'none'}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]['_id'], str(publicJob['_id'])) def testListAllJobs(self): self.jobModel.createJob(title='user 0 job', type='t', user=self.users[0], public=False) self.jobModel.createJob(title='user 1 job', type='t', user=self.users[1], public=False) self.jobModel.createJob(title='user 1 job', type='t', user=self.users[1], public=True) self.jobModel.createJob(title='user 2 job', type='t', user=self.users[2]) self.jobModel.createJob(title='anonymous job', type='t') self.jobModel.createJob(title='anonymous public job', type='t2', public=True) # User 0, as a site admin, should be able to see all jobs resp = self.request('/job/all', user=self.users[0]) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 6) # get with filter resp = self.request('/job/all', user=self.users[0], params={ 'types': json.dumps(['t']), 'statuses': json.dumps([0]) }) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 5) # get with unmet filter conditions resp = self.request('/job/all', user=self.users[0], params={ 'types': json.dumps(['nonexisttype']) }) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 0) # User 1, as non site admin, should encounter http 403 (Forbidden) resp = self.request('/job/all', user=self.users[1]) self.assertStatus(resp, 403) # Not authenticated user should encounter http 401 (unauthorized) resp = self.request('/job/all') self.assertStatus(resp, 401) def testFiltering(self): job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=True) job['_some_other_field'] = 'foo' job = self.jobModel.save(job) resp = self.request('/job/%s' % job['_id']) self.assertStatusOk(resp) self.assertTrue('created' in resp.json) self.assertTrue('_some_other_field' not in resp.json) self.assertTrue('kwargs' not in resp.json) self.assertTrue('args' not in resp.json) resp = self.request('/job/%s' % job['_id'], user=self.users[0]) self.assertTrue('kwargs' in resp.json) self.assertTrue('args' in resp.json) self.jobModel.exposeFields(level=AccessType.READ, fields={'_some_other_field'}) self.jobModel.hideFields(level=AccessType.READ, fields={'created'}) resp = self.request('/job/%s' % job['_id']) self.assertStatusOk(resp) self.assertEqual(resp.json['_some_other_field'], 'foo') self.assertTrue('created' not in resp.json) def testJobProgressAndNotifications(self): job = self.jobModel.createJob(title='a job', type='t', user=self.users[1], public=True) path = '/job/%s' % job['_id'] resp = self.request(path) self.assertEqual(resp.json['progress'], None) self.assertEqual(resp.json['timestamps'], []) resp = self.request(path, method='PUT', user=self.users[1], params={ 'progressTotal': 100, 'progressCurrent': 3, 'progressMessage': 'Started', 'notify': 'false', 'status': JobStatus.QUEUED }) self.assertStatusOk(resp) self.assertEqual(resp.json['progress'], { 'total': 100, 'current': 3, 'message': 'Started', 'notificationId': None }) # The status update should make it so we now have a timestamp self.assertEqual(len(resp.json['timestamps']), 1) self.assertEqual(resp.json['timestamps'][0]['status'], JobStatus.QUEUED) self.assertIn('time', resp.json['timestamps'][0]) # If the status does not change on update, no timestamp should be added resp = self.request(path, method='PUT', user=self.users[1], params={ 'status': JobStatus.QUEUED }) self.assertStatusOk(resp) self.assertEqual(len(resp.json['timestamps']), 1) self.assertEqual(resp.json['timestamps'][0]['status'], JobStatus.QUEUED) # We passed notify=false, so we should only have the job creation notification resp = self.request(path='/notification/stream', method='GET', user=self.users[1], isJson=False, params={'timeout': 0}) messages = self.getSseMessages(resp) self.assertEqual(len(messages), 1) # Update progress with notify=true (the default) resp = self.request(path, method='PUT', user=self.users[1], params={ 'progressCurrent': 50, 'progressMessage': 'Something bad happened', 'status': JobStatus.ERROR }) self.assertStatusOk(resp) self.assertNotEqual(resp.json['progress']['notificationId'], None) # We should now see three notifications (job created + job status + progress) resp = self.request(path='/notification/stream', method='GET', user=self.users[1], isJson=False, params={'timeout': 0}) messages = self.getSseMessages(resp) job = self.jobModel.load(job['_id'], force=True) self.assertEqual(len(messages), 3) creationNotify = messages[0] progressNotify = messages[1] statusNotify = messages[2] self.assertEqual(creationNotify['type'], 'job_created') self.assertEqual(creationNotify['data']['_id'], str(job['_id'])) self.assertEqual(statusNotify['type'], 'job_status') self.assertEqual(statusNotify['data']['_id'], str(job['_id'])) self.assertEqual(int(statusNotify['data']['status']), JobStatus.ERROR) self.assertNotIn('kwargs', statusNotify['data']) self.assertNotIn('log', statusNotify['data']) self.assertEqual(progressNotify['type'], 'progress') self.assertEqual(progressNotify['data']['title'], job['title']) self.assertEqual(progressNotify['data']['current'], float(50)) self.assertEqual(progressNotify['data']['state'], 'error') self.assertEqual(progressNotify['_id'], str(job['progress']['notificationId'])) def testDotsInKwargs(self): kwargs = { '$key.with.dots': 'value', 'foo': [{ 'moar.dots': True }] } job = self.jobModel.createJob(title='dots', type='x', user=self.users[0], kwargs=kwargs) # Make sure we can update a job and notification creation works self.jobModel.updateJob(job, status=JobStatus.QUEUED, notify=True) self.assertEqual(job['kwargs'], kwargs) resp = self.request('/job/%s' % job['_id'], user=self.users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json['kwargs'], kwargs) job = self.jobModel.load(job['_id'], force=True) self.assertEqual(job['kwargs'], kwargs) job = self.jobModel.filter(job, self.users[0]) self.assertEqual(job['kwargs'], kwargs) job = self.jobModel.filter(job, self.users[1]) self.assertFalse('kwargs' in job) def testLocalJob(self): job = self.jobModel.createLocalJob( title='local', type='local', user=self.users[0], kwargs={ 'hello': 'world' }, module='plugin_tests.local_job_impl') self.jobModel.scheduleJob(job) job = self.jobModel.load(job['_id'], force=True, includeLog=True) self.assertEqual(job['log'], ['job ran!']) job = self.jobModel.createLocalJob( title='local', type='local', user=self.users[0], kwargs={ 'hello': 'world' }, module='plugin_tests.local_job_impl', function='fail') self.jobModel.scheduleJob(job) job = self.jobModel.load(job['_id'], force=True, includeLog=True) self.assertEqual(job['log'], ['job failed']) def testValidateCustomStatus(self): job = self.jobModel.createJob(title='test', type='x', user=self.users[0]) def validateStatus(event): if event.info == 1234: event.preventDefault().addResponse(True) def validTransitions(event): if event.info['status'] == 1234: event.preventDefault().addResponse([JobStatus.INACTIVE]) with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status=1234) # Should fail with events.bound('jobs.status.validate', 'test', validateStatus), \ events.bound('jobs.status.validTransitions', 'test', validTransitions): self.jobModel.updateJob(job, status=1234) # Should work with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status=4321) # Should fail def testValidateCustomStrStatus(self): job = self.jobModel.createJob(title='test', type='x', user=self.users[0]) def validateStatus(event): states = ['a', 'b', 'c'] if event.info in states: event.preventDefault().addResponse(True) def validTransitions(event): if event.info['status'] == 'a': event.preventDefault().addResponse([JobStatus.INACTIVE]) with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status='a') with events.bound('jobs.status.validate', 'test', validateStatus), \ events.bound('jobs.status.validTransitions', 'test', validTransitions): self.jobModel.updateJob(job, status='a') self.assertEqual(job['status'], 'a') with self.assertRaises(ValidationException), \ events.bound('jobs.status.validate', 'test', validateStatus): self.jobModel.updateJob(job, status='foo') def testUpdateOtherFields(self): job = self.jobModel.createJob(title='test', type='x', user=self.users[0]) job = self.jobModel.updateJob(job, otherFields={'other': 'fields'}) self.assertEqual(job['other'], 'fields') def testCancelJob(self): job = self.jobModel.createJob(title='test', type='x', user=self.users[0]) # add to the log job = self.jobModel.updateJob(job, log='entry 1\n') # Reload without the log job = self.jobModel.load(id=job['_id'], force=True) self.assertEqual(len(job.get('log', [])), 0) # Cancel job = self.jobModel.cancelJob(job) self.assertEqual(job['status'], JobStatus.CANCELED) # Reloading should still have the log and be canceled job = self.jobModel.load(id=job['_id'], force=True, includeLog=True) self.assertEqual(job['status'], JobStatus.CANCELED) self.assertEqual(len(job.get('log', [])), 1) def testCancelJobEndpoint(self): job = self.jobModel.createJob(title='test', type='x', user=self.users[0]) # Ensure requires write perms jobCancelUrl = '/job/%s/cancel' % job['_id'] resp = self.request(jobCancelUrl, user=self.users[1], method='PUT') self.assertStatus(resp, 403) # Try again with the right user jobCancelUrl = '/job/%s/cancel' % job['_id'] resp = self.request(jobCancelUrl, user=self.users[0], method='PUT') self.assertStatusOk(resp) self.assertEqual(resp.json['status'], JobStatus.CANCELED) def testJobsTypesAndStatuses(self): self.jobModel.createJob(title='user 0 job', type='t1', user=self.users[0], public=False) self.jobModel.createJob(title='user 1 job', type='t2', user=self.users[1], public=False) self.jobModel.createJob(title='user 1 job', type='t3', user=self.users[1], public=True) self.jobModel.createJob(title='user 2 job', type='t4', user=self.users[2]) self.jobModel.createJob(title='anonymous job', type='t5') self.jobModel.createJob(title='anonymous public job', type='t6', public=True) # User 1, as non site admin, should encounter http 403 (Forbidden) resp = self.request('/job/typeandstatus/all', user=self.users[1]) self.assertStatus(resp, 403) # Admin user gets all types and statuses resp = self.request('/job/typeandstatus/all', user=self.users[0]) self.assertStatusOk(resp) self.assertEqual(len(resp.json['types']), 6) self.assertEqual(len(resp.json['statuses']), 1) # standard user gets types and statuses of its own jobs resp = self.request('/job/typeandstatus', user=self.users[1]) self.assertStatusOk(resp) self.assertEqual(len(resp.json['types']), 2) self.assertEqual(len(resp.json['statuses']), 1) def testDefaultParentId(self): job = self.jobModel.createJob(title='Job', type='Job', user=self.users[0]) # If not specified parentId should be None self.assertEquals(job['parentId'], None) def testIsParentIdCorrect(self): parentJob = self.jobModel.createJob( title='Parent Job', type='Parent Job', user=self.users[0]) childJob = self.jobModel.createJob( title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) # During initialization parent job should be set correctly self.assertEqual(childJob['parentId'], parentJob['_id']) def testSetParentCorrectly(self): parentJob = self.jobModel.createJob( title='Parent Job', type='Parent Job', user=self.users[0]) childJob = self.jobModel.createJob(title='Child Job', type='Child Job', user=self.users[0]) self.jobModel.setParentJob(childJob, parentJob) # After setParentJob method is called parent job should be set correctly self.assertEqual(childJob['parentId'], parentJob['_id']) def testParentCannotBeEqualToChild(self): childJob = self.jobModel.createJob(title='Child Job', type='Child Job', user=self.users[0]) # Cannot set a job as it's own parent with self.assertRaises(ValidationException): self.jobModel.setParentJob(childJob, childJob) def testParentIdCannotBeOverridden(self): parentJob = self.jobModel.createJob( title='Parent Job', type='Parent Job', user=self.users[0]) anotherParentJob = self.jobModel.createJob( title='Another Parent Job', type='Parent Job', user=self.users[0]) childJob = self.jobModel.createJob( title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) with self.assertRaises(ValidationException): # If parent job is set, cannot be overridden self.jobModel.setParentJob(childJob, anotherParentJob) def testListChildJobs(self): parentJob = self.jobModel.createJob( title='Parent Job', type='Parent Job', user=self.users[0]) childJob = self.jobModel.createJob( title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) self.jobModel.createJob( title='Another Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) # Should return a list with 2 jobs self.assertEquals(len(list(self.jobModel.listChildJobs(parentJob))), 2) # Should return an empty list self.assertEquals(len(list(self.jobModel.listChildJobs(childJob))), 0) def testListChildJobsRest(self): parentJob = self.jobModel.createJob( title='Parent Job', type='Parent Job', user=self.users[0]) childJob = self.jobModel.createJob( title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) self.jobModel.createJob( title='Another Child Job', type='Child Job', user=self.users[0], parentJob=parentJob) resp = self.request('/job', user=self.users[0], params={'parentId': str(parentJob['_id'])}) resp2 = self.request('/job', user=self.users[0], params={'parentId': str(childJob['_id'])}) self.assertStatusOk(resp) self.assertStatusOk(resp2) # Should return a list with 2 jobs self.assertEquals(len(resp.json), 2) # Should return an empty list self.assertEquals(len(resp2.json), 0) def testCreateJobRest(self): resp = self.request('/job', method='POST', user=self.users[0], params={'title': 'job', 'type': 'job'}) # If user does not have the necessary token status is 403 self.assertStatus(resp, 403) token = Token().createToken(scope=REST_CREATE_JOB_TOKEN_SCOPE) resp2 = self.request( '/job', method='POST', token=token, params={'title': 'job', 'type': 'job'}) # If user has the necessary token status is 200 self.assertStatusOk(resp2) def testJobStateTransitions(self): job = self.jobModel.createJob( title='user 0 job', type='t1', user=self.users[0], public=False) # We can't move straight to SUCCESS with self.assertRaises(ValidationException): job = self.jobModel.updateJob(job, status=JobStatus.SUCCESS) self.jobModel.updateJob(job, status=JobStatus.QUEUED) self.jobModel.updateJob(job, status=JobStatus.RUNNING) self.jobModel.updateJob(job, status=JobStatus.ERROR) # We shouldn't be able to move backwards with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status=JobStatus.QUEUED) with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status=JobStatus.RUNNING) with self.assertRaises(ValidationException): self.jobModel.updateJob(job, status=JobStatus.INACTIVE) def testJobSaveEventModification(self): def customSave(event): kwargs = json_util.loads(event.info['kwargs']) kwargs['key2'] = 'newvalue' event.info['kwargs'] = json_util.dumps(kwargs) job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=True) job['kwargs'] = {'key1': 'value1', 'key2': 'value2'} with events.bound('model.job.save', 'test', customSave): job = self.jobModel.save(job) self.assertEqual(job['kwargs']['key2'], 'newvalue')
def testConfigureItemTaskFromSlicerCli(self): # Create a new item that will become a task item = Item().createItem(name='placeholder', creator=self.admin, folder=self.privateFolder) # Create task to introspect container with mock.patch('girder_jobs.models.job.Job.scheduleJob') as scheduleMock: resp = self.request( '/item/%s/item_task_slicer_cli_description' % item['_id'], method='POST', params={ 'image': 'johndoe/foo:v5', 'args': json.dumps(['--foo', 'bar']) }, user=self.admin) self.assertStatusOk(resp) self.assertEqual(resp.json['_modelType'], 'job') self.assertEqual(len(scheduleMock.mock_calls), 1) job = scheduleMock.mock_calls[0][1][0] self.assertEqual(job['handler'], 'worker_handler') self.assertEqual(job['itemTaskId'], item['_id']) self.assertEqual(job['kwargs']['outputs']['_stdout']['method'], 'PUT') self.assertTrue(job['kwargs']['outputs']['_stdout']['url'].endswith( 'item/%s/item_task_slicer_cli_xml' % item['_id'])) token = job['kwargs']['outputs']['_stdout']['headers']['Girder-Token'] # Task should not be registered until we get the callback resp = self.request('/item_task', user=self.admin) self.assertStatusOk(resp) self.assertEqual(resp.json, []) # Image and args should be stored in the item metadata item = Item().load(item['_id'], force=True) self.assertEqual(item['meta']['itemTaskSpec']['docker_image'], 'johndoe/foo:v5') self.assertEqual(item['meta']['itemTaskSlicerCliArgs'], ['--foo', 'bar']) # Simulate callback from introspection job with open(os.path.join(os.path.dirname(__file__), 'slicer_cli.xml')) as f: xml = f.read() resp = self.request( '/item/%s/item_task_slicer_cli_xml' % item['_id'], method='PUT', params={ 'setName': True, 'setDescription': True }, token=token, body=xml, type='application/xml') self.assertStatusOk(resp) # We should only be able to see tasks we have read access on resp = self.request('/item_task') self.assertStatusOk(resp) self.assertEqual(resp.json, []) resp = self.request('/item_task', user=self.admin) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]['_id'], str(item['_id'])) item = Item().load(item['_id'], force=True) self.assertEqual(item['name'], 'PET phantom detector CLI') self.assertEqual( item['description'], u'**Description**: Detects positions of PET/CT pocket phantoms in PET image.\n\n' u'**Author(s)**: Girder Developers\n\n**Version**: 1.0\n\n' u'**License**: Apache 2.0\n\n**Acknowledgements**: *none*\n\n' u'*This description was auto-generated from the Slicer CLI XML specification.*' ) self.assertTrue(item['meta']['isItemTask']) self.assertEqual(item['meta']['itemTaskSpec'], { 'mode': 'docker', 'docker_image': 'johndoe/foo:v5', 'container_args': [ '--foo', 'bar', '--InputImage=$input{--InputImage}', '--MaximumLineStraightnessDeviation=$input{--MaximumLineStraightnessDeviation}', '--MaximumRadius=$input{--MaximumRadius}', '--MaximumSphereDistance=$input{--MaximumSphereDistance}', '--MinimumRadius=$input{--MinimumRadius}', '--MinimumSphereActivity=$input{--MinimumSphereActivity}', '--MinimumSphereDistance=$input{--MinimumSphereDistance}', '--SpheresPerPhantom=$input{--SpheresPerPhantom}', '$flag{--StrictSorting}', '--DetectedPoints=$output{--DetectedPoints}' ], 'inputs': [{ 'description': 'Input image to be analysed.', 'format': 'image', 'name': 'InputImage', 'type': 'image', 'id': '--InputImage', 'target': 'filepath' }, { 'description': 'Used for eliminating detections which are not in a straight line. ' 'Unit: multiples of geometric average of voxel spacing', 'format': 'number', 'default': {'data': 1.0}, 'type': 'number', 'id': '--MaximumLineStraightnessDeviation', 'name': 'MaximumLineStraightnessDeviation' }, { 'description': 'Used for eliminating too big blobs. Unit: millimeter [mm]', 'format': 'number', 'default': {'data': 20.0}, 'type': 'number', 'id': '--MaximumRadius', 'name': 'MaximumRadius' }, { 'description': 'Signifies maximum distance between adjacent sphere centers [mm]. ' 'Used to separate phantoms from tumors.', 'format': 'number', 'default': {'data': 40.0}, 'type': 'number', 'id': '--MaximumSphereDistance', 'name': 'MaximumSphereDistance' }, { 'description': 'Used for eliminating too small blobs. Unit: millimeter [mm]', 'format': 'number', 'default': {'data': 3.0}, 'type': 'number', 'id': '--MinimumRadius', 'name': 'MinimumRadius' }, { 'description': 'Used for thresholding in blob detection. ' 'Unit: becquerels per milliliter [Bq/ml]', 'format': 'number', 'default': {'data': 5000.0}, 'type': 'number', 'id': '--MinimumSphereActivity', 'name': 'MinimumSphereActivity' }, { 'description': 'Signifies minimum distance between adjacent sphere centers [mm]. ' 'Used to separate phantoms from tumors.', 'format': 'number', 'default': {'data': 30.0}, 'type': 'number', 'id': '--MinimumSphereDistance', 'name': 'MinimumSphereDistance' }, { 'description': 'What kind of phantom are we working with here?', 'format': 'number-enumeration', 'default': {'data': 3}, 'type': 'number-enumeration', 'id': '--SpheresPerPhantom', 'name': 'SpheresPerPhantom', 'values': [2, 3] }, { 'description': 'Controls whether spheres within a phantom must have descending ' 'activities. If OFF, they can have approximately same activities ' '(within 15%).', 'format': 'boolean', 'default': {'data': False}, 'type': 'boolean', 'id': '--StrictSorting', 'name': 'StrictSorting' }], 'outputs': [{ 'description': 'Fiducial points, one for each detected sphere. ' 'Will be multiple of 3.', 'format': 'new-file', 'name': 'DetectedPoints', 'type': 'new-file', 'id': '--DetectedPoints', 'target': 'filepath' }] }) # Shouldn't be able to run the task if we don't have execute permission flag Folder().setUserAccess( self.privateFolder, user=self.user, level=AccessType.READ, save=True) resp = self.request( '/item_task/%s/execution' % item['_id'], method='POST', user=self.user) self.assertStatus(resp, 403) # Grant the user permission, and run the task Folder().setUserAccess( self.privateFolder, user=self.user, level=AccessType.WRITE, flags=ACCESS_FLAG_EXECUTE_TASK, currentUser=self.admin, save=True) inputs = { '--InputImage': { 'mode': 'girder', 'resource_type': 'item', 'id': str(item['_id']) }, '--MaximumLineStraightnessDeviation': { 'mode': 'inline', 'data': 1 }, '--MaximumRadius': { 'mode': 'inline', 'data': 20 }, '--MaximumSphereDistance': { 'mode': 'inline', 'data': 40 }, '--MinimumRadius': { 'mode': 'inline', 'data': 3 }, '--MinimumSphereActivity': { 'mode': 'inline', 'data': 5000 }, '--MinimumSphereDistance': { 'mode': 'inline', 'data': 30 }, '--SpheresPerPhantom': { 'mode': 'inline', 'data': 3}, '--StrictSorting': { 'mode': 'inline', 'data': False } } outputs = { '--DetectedPoints': { 'mode': 'girder', 'parent_id': str(self.privateFolder['_id']), 'parent_type': 'folder', 'name': 'test.txt' } } # Ensure task was scheduled with mock.patch('girder_jobs.models.job.Job.scheduleJob') as scheduleMock: resp = self.request( '/item_task/%s/execution' % item['_id'], method='POST', user=self.user, params={ 'inputs': json.dumps(inputs), 'outputs': json.dumps(outputs) }) self.assertEqual(len(scheduleMock.mock_calls), 1) self.assertStatusOk(resp) job = resp.json self.assertEqual(job['_modelType'], 'job') self.assertNotIn('kwargs', job) # ordinary user can't see kwargs jobModel = Job() job = jobModel.load(job['_id'], force=True) output = job['kwargs']['outputs']['--DetectedPoints'] # Simulate output from the worker contents = b'Hello world' resp = self.request( path='/file', method='POST', token=output['token'], params={ 'parentType': output['parent_type'], 'parentId': output['parent_id'], 'name': output['name'], 'size': len(contents), 'mimeType': 'text/plain', 'reference': output['reference'] }) self.assertStatusOk(resp) uploadId = resp.json['_id'] fields = [('offset', 0), ('uploadId', uploadId)] files = [('chunk', output['name'], contents)] resp = self.multipartRequest( path='/file/chunk', fields=fields, files=files, token=output['token']) self.assertStatusOk(resp) file = resp.json self.assertEqual(file['_modelType'], 'file') self.assertEqual(file['size'], 11) self.assertEqual(file['mimeType'], 'text/plain') file = File().load(file['_id'], force=True) # Make sure temp token is removed once we change job status to final state job = jobModel.load(job['_id'], force=True) self.assertIn('itemTaskTempToken', job) # Transition through states to SUCCESS job = jobModel.updateJob(job, status=JobStatus.QUEUED) job = jobModel.updateJob(job, status=JobStatus.RUNNING) job = jobModel.updateJob(job, status=JobStatus.SUCCESS) self.assertNotIn('itemTaskTempToken', job) self.assertIn('itemTaskBindings', job) # Wait for async data.process event to bind output provenance start = time.time() while time.time() - start < 15: job = jobModel.load(job['_id'], force=True) if 'itemId' in job['itemTaskBindings']['outputs']['--DetectedPoints']: break else: time.sleep(0.2) else: raise Exception('Output binding did not occur in time') self.assertEqual( job['itemTaskBindings']['outputs']['--DetectedPoints']['itemId'], file['itemId'])