def testWorkerStatusEndpoint(server, models):
# Create a job to be handled by the worker plugin
job = Job().createJob(
title='title', type='foo', handler='worker_handler',
user=models['admin'], public=False, args=(), kwargs={})
job['kwargs'] = {
'jobInfo': utils.jobInfoSpec(job),
'inputs': [
utils.girderInputSpec(models['adminFolder'], resourceType='folder')
],
'outputs': [
utils.girderOutputSpec(models['adminFolder'], token=models['adminToken'])
]
}
job = Job().save(job)
assert job['status'] == JobStatus.INACTIVE
# Schedule the job
with mock.patch('celery.Celery') as celeryMock:
instance = celeryMock.return_value
instance.send_task.return_value = FakeAsyncResult()
Job().scheduleJob(job)
# Call the worker status endpoint
resp = server.request('/worker/status', method='GET', user=models['admin'])
assertStatusOk(resp)
for key in ['report', 'stats', 'ping', 'active', 'reserved']:
assert key in resp.json
def load(self, info):
getPlugin('jobs').load(info)
info['apiRoot'].nli = NLI()
events.bind('jobs.job.update.after', 'nlisim', update_status)
job_model = Job()
job_model.exposeFields(level=constants.AccessType.ADMIN, fields={'args', 'kwargs'})
def cache_tile_frames_job(job):
from girder_jobs.constants import JobStatus
from girder_jobs.models.job import Job
from girder_large_image.models.image_item import ImageItem
from girder import logger
kwargs = job['kwargs']
item = ImageItem().load(kwargs.pop('itemId'), force=True)
job = Job().updateJob(job,
log='Started caching tile frames\n',
status=JobStatus.RUNNING)
try:
for entry in kwargs.get('tileFramesList'):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.CANCELED:
return
job = Job().updateJob(job, log='Caching %r\n' % entry)
ImageItem().tileFrames(item, checkAndCreate=True, **entry)
job = Job().updateJob(job,
log='Finished caching tile frames\n',
status=JobStatus.SUCCESS)
except Exception as exc:
logger.exception('Failed caching tile frames')
job = Job().updateJob(job,
log='Failed caching tile frames (%s)\n' % exc,
status=JobStatus.ERROR)
def schedule(event):
"""
This is bound to the "jobs.schedule" event, and will be triggered any time
a job is scheduled. This handler will process any job that has the
handler field set to "worker_handler".
"""
job = event.info
if job['handler'] == 'worker_handler':
task = job.get('celeryTaskName', 'girder_worker.run')
# Set the job status to queued
Job().updateJob(job, status=JobStatus.QUEUED)
# Send the task to celery
asyncResult = getCeleryApp().send_task(task,
job['args'],
job['kwargs'],
queue=job.get('celeryQueue'),
headers={
'jobInfoSpec':
jobInfoSpec(
job,
job.get('token', None)),
'apiUrl':
getWorkerApiUrl()
})
# Record the task ID from celery.
Job().updateJob(job, otherFields={'celeryTaskId': asyncResult.task_id})
# Stop event propagation since we have taken care of scheduling.
event.stopPropagation()
def scheduleThumbnailJob(file,
attachToType,
attachToId,
user,
width=0,
height=0,
crop=True):
"""
Schedule a local thumbnail creation job and return it.
"""
job = Job().createLocalJob(title='Generate thumbnail for %s' %
file['name'],
user=user,
type='thumbnails.create',
public=False,
module='girder_thumbnails.worker',
kwargs={
'fileId': str(file['_id']),
'width': width,
'height': height,
'crop': crop,
'attachToType': attachToType,
'attachToId': str(attachToId)
})
Job().scheduleJob(job)
return job
def createThumbnails(self, params):
self.requireParams(['spec'], params)
try:
spec = json.loads(params['spec'])
if not isinstance(spec, list):
raise ValueError()
except ValueError:
raise RestException('The spec parameter must be a JSON list.')
maxThumbnailFiles = int(Setting().get(
constants.PluginSettings.LARGE_IMAGE_MAX_THUMBNAIL_FILES))
if maxThumbnailFiles <= 0:
raise RestException('Thumbnail files are not enabled.')
jobKwargs = {'spec': spec}
if params.get('logInterval') is not None:
jobKwargs['logInterval'] = float(params['logInterval'])
if params.get('concurrent') is not None:
jobKwargs['concurrent'] = float(params['concurrent'])
job = Job().createLocalJob(
module='girder_large_image.rest.large_image_resource',
function='createThumbnailsJob',
kwargs=jobKwargs,
title='Create large image thumbnail files.',
type='large_image_create_thumbnails',
user=self.getCurrentUser(),
public=True,
asynchronous=True,
)
Job().scheduleJob(job)
return job
def _onUpload(event):
"""
Look at uploads containing references related to this plugin. If found,
they are used to link item task outputs back to a job document.
"""
try:
ref = json.loads(event.info.get('reference', ''))
except ValueError:
return
if isinstance(ref, dict) and ref.get('type') == 'item_tasks.output':
jobModel = Job()
tokenModel = Token()
token = event.info['currentToken']
if tokenModel.hasScope(token, 'item_tasks.job_write:%s' % ref['jobId']):
job = jobModel.load(ref['jobId'], force=True, exc=True)
else:
job = jobModel.load(
ref['jobId'], level=AccessType.WRITE, user=event.info['currentUser'], exc=True)
file = event.info['file']
item = Item().load(file['itemId'], force=True)
# Add link to job model to the output item
jobModel.updateJob(job, otherFields={
'itemTaskBindings.outputs.%s.itemId' % ref['id']: item['_id']
})
# Also a link in the item to the job that created it
item['createdByJob'] = job['_id']
Item().save(item)
def convertImage(self,
item,
fileObj,
user=None,
token=None,
localJob=True,
**kwargs):
if fileObj['itemId'] != item['_id']:
raise TileGeneralException(
'The provided file must be in the provided item.')
if not localJob:
return self._convertImageViaWorker(item, fileObj, user, token,
**kwargs)
# local job
job = Job().createLocalJob(
module='large_image_tasks.tasks',
function='convert_image_job',
kwargs={
'itemId': str(item['_id']),
'fileId': str(fileObj['_id']),
'userId': str(user['_id']) if user else None,
**kwargs,
},
title='Convert a file to a large image file.',
type='large_image_convert_image',
user=user,
public=True,
asynchronous=True,
)
Job().scheduleJob(job)
return job
def setUp(self):
base.TestCase.setUp(self)
self.users = [User().createUser(
'usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n)
for n in range(3)]
self.jobModel = Job()
def testDeleteIncompleteTile(server, admin, user, fsAssetstore,
unavailableWorker): # noqa
# Test the large_image/settings end point
resp = server.request(method='DELETE',
path='/large_image/tiles/incomplete',
user=user)
assert utilities.respStatus(resp) == 403
resp = server.request(method='DELETE',
path='/large_image/tiles/incomplete',
user=admin)
assert utilities.respStatus(resp) == 200
results = resp.json
assert results['removed'] == 0
file = utilities.uploadTestFile('yb10kx5k.png', admin, fsAssetstore)
itemId = str(file['itemId'])
resp = server.request(method='POST',
path='/item/%s/tiles' % itemId,
user=admin)
resp = server.request(method='DELETE',
path='/large_image/tiles/incomplete',
user=admin)
assert utilities.respStatus(resp) == 200
results = resp.json
assert results['removed'] == 1
def preventCancel(evt):
job = evt.info['job']
params = evt.info['params']
if (params.get('status') and params.get('status') != job['status']
and params['status']
in (JobStatus.CANCELED, CustomJobStatus.CANCELING)):
evt.preventDefault()
# Prevent a job from cancelling
events.bind('jobs.job.update', 'testDeleteIncompleteTile', preventCancel)
# Create a job and mark it as running
resp = server.request(method='POST',
path='/item/%s/tiles' % itemId,
user=admin)
job = Job().load(id=resp.json['_id'], force=True)
Job().updateJob(job, status=JobStatus.RUNNING)
resp = server.request(method='DELETE',
path='/large_image/tiles/incomplete',
user=admin)
events.unbind('jobs.job.update', 'testDeleteIncompleteTile')
assert utilities.respStatus(resp) == 200
results = resp.json
assert results['removed'] == 0
assert 'could not be canceled' in results['message']
# Now we should be able to cancel the job
resp = server.request(method='DELETE',
path='/large_image/tiles/incomplete',
user=admin)
assert utilities.respStatus(resp) == 200
results = resp.json
assert results['removed'] == 1
def setUp(self):
super().setUp()
self.users = [
User().createUser('usr' + str(n), 'passwd', 'tst', 'usr',
'*****@*****.**' % n) for n in range(3)
]
self.jobModel = Job()
def convert_image_job(job):
import tempfile
from girder.constants import AccessType
from girder.models.file import File
from girder.models.folder import Folder
from girder.models.item import Item
from girder.models.upload import Upload
from girder.models.user import User
from girder_jobs.constants import JobStatus
from girder_jobs.models.job import Job
kwargs = job['kwargs']
item = Item().load(kwargs.pop('itemId'), force=True)
fileObj = File().load(kwargs.pop('fileId'), force=True)
userId = kwargs.pop('userId', None)
user = User().load(userId, force=True) if userId else None
folder = Folder().load(kwargs.pop('folderId', item['folderId']),
user=user, level=AccessType.WRITE)
name = kwargs.pop('name', None)
job = Job().updateJob(
job, log='Started large image conversion\n',
status=JobStatus.RUNNING)
logger = logging.getLogger('large-image-converter')
handler = JobLogger(job=job)
logger.addHandler(handler)
# We could increase the default logging level here
# logger.setLevel(logging.DEBUG)
try:
with tempfile.TemporaryDirectory() as tempdir:
dest = create_tiff(
inputFile=File().getLocalFilePath(fileObj),
inputName=fileObj['name'],
outputDir=tempdir,
**kwargs,
)
job = Job().updateJob(job, log='Storing result\n')
with open(dest, 'rb') as fobj:
Upload().uploadFromFile(
fobj,
size=os.path.getsize(dest),
name=name or os.path.basename(dest),
parentType='folder',
parent=folder,
user=user,
)
except Exception as exc:
status = JobStatus.ERROR
logger.exception('Failed in large image conversion')
job = Job().updateJob(
job, log='Failed in large image conversion (%s)\n' % exc, status=status)
else:
status = JobStatus.SUCCESS
job = Job().updateJob(
job, log='Finished large image conversion\n', status=status)
finally:
logger.removeHandler(handler)
def testLocalJob(models):
# Make sure local jobs still work
job = Job().createLocalJob(
title='local', type='local', user=models['admin'],
module='plugin_tests.worker_test', function='local_job')
Job().scheduleJob(job)
job = Job().load(job['_id'], force=True, includeLog=True)
assert 'job ran' in job['log']
def testWorkerCancel(models):
jobModel = Job()
job = jobModel.createJob(
title='title', type='foo', handler='worker_handler',
user=models['admin'], public=False, args=(), kwargs={})
job['kwargs'] = {
'jobInfo': utils.jobInfoSpec(job),
'inputs': [
utils.girderInputSpec(models['adminFolder'], resourceType='folder')
],
'outputs': [
utils.girderOutputSpec(models['adminFolder'], token=models['adminToken'])
]
}
job = jobModel.save(job)
assert job['status'] == JobStatus.INACTIVE
# Schedule the job, make sure it is sent to celery
with mock.patch('celery.Celery') as celeryMock, \
mock.patch('girder_worker.girder_plugin.event_handlers.AsyncResult') as asyncResult:
instance = celeryMock.return_value
instance.send_task.return_value = FakeAsyncResult()
jobModel.scheduleJob(job)
jobModel.cancelJob(job)
asyncResult.assert_called_with('fake_id', app=mock.ANY)
# Check we called revoke
asyncResult.return_value.revoke.assert_called_once()
job = jobModel.load(job['_id'], force=True)
assert job['status'] == CustomJobStatus.CANCELING
def execute_simulation(self, name, config, folder=None):
target_time = config.get('simulation', {}).get('run_time', 50)
user, token = self.getCurrentUser(returnToken=True)
folder_model = Folder()
job_model = Job()
if folder is None:
folder = folder_model.findOne(
{'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user'}
)
if folder is None:
raise RestException('Could not find the user\'s "public" folder.')
simulation_model = Simulation()
simulation = simulation_model.createSimulation(
folder,
name,
config,
user,
True,
)
girder_config = GirderConfig(
api=GIRDER_API, token=str(token['_id']), folder=str(folder['_id'])
)
simulation_config = SimulationConfig(NLI_CONFIG_FILE, config)
# TODO: This would be better stored as a dict, but it's easier once we change the
# config object format.
simulation_config_file = StringIO()
simulation_config.write(simulation_config_file)
job = job_model.createJob(
title='NLI Simulation',
type=NLI_JOB_TYPE,
kwargs={
'girder_config': attr.asdict(girder_config),
'simulation_config': simulation_config_file.getvalue(),
'config': config,
'simulation_id': simulation['_id'],
},
user=user,
)
simulation['nli']['job_id'] = job['_id']
simulation_model.save(simulation)
run_simulation.delay(
name=name,
girder_config=girder_config,
simulation_config=simulation_config,
target_time=target_time,
job=job,
simulation_id=simulation['_id'],
)
return job
def testWorkerWithParent(models):
jobModel = Job()
parentJob = jobModel.createJob(
title='title', type='foo', handler='worker_handler',
user=models['admin'], public=False, otherFields={'celeryTaskId': '1234'})
childJob = jobModel.createJob(
title='title', type='foo', handler='worker_handler',
user=models['admin'], public=False, otherFields={'celeryTaskId': '5678',
'celeryParentTaskId': '1234'})
assert parentJob['_id'] == childJob['parentId']
def import_recursive(job):
try:
root = job['kwargs']['root']
token = job['kwargs']['token']
user = User().load(job['userId'], force=True)
children = list(Folder().childFolders(root, 'collection', user=user))
count = len(children)
progress = 0
job = Job().updateJob(job,
log='Started TCGA import\n',
status=JobStatus.RUNNING,
progressCurrent=progress,
progressTotal=count)
logger.info('Starting recursive TCGA import')
for child in children:
progress += 1
try:
msg = 'Importing "%s"' % child.get('name', '')
job = Job().updateJob(job,
log=msg,
progressMessage=msg + '\n',
progressCurrent=progress)
logger.debug(msg)
Cohort().importDocument(child,
recurse=True,
user=user,
token=token,
job=job)
job = Job().load(id=job['_id'], force=True)
# handle any request to stop execution
if (not job or job['status']
in (JobStatus.CANCELED, JobStatus.ERROR)):
logger.info('TCGA import job halted with')
return
except ValidationException:
logger.warning('Failed to import %s' % child.get('name', ''))
logger.info('Starting recursive TCGA import')
job = Job().updateJob(job,
log='Finished TCGA import\n',
status=JobStatus.SUCCESS,
progressCurrent=count,
progressMessage='Finished TCGA import')
except Exception as e:
logger.exception('Importing TCGA failed with %s' % str(e))
job = Job().updateJob(job,
log='Import failed with %s\n' % str(e),
status=JobStatus.ERROR)
def list_simulation_jobs(self, limit, offset, sort):
user = self.getCurrentUser()
job_model = Job()
return job_model.list(
types=[NLI_JOB_TYPE],
statuses=[JobStatus.QUEUED, JobStatus.RUNNING],
user=user,
currentUser=user,
limit=limit,
offset=offset,
sort=sort,
)
def run_pipeline_task(self, folder, pipeline: PipelineDescription):
"""
Run a pipeline on a dataset.
:param folder: The girder folder containing the dataset to run on.
:param pipeline: The pipeline to run the dataset on.
"""
folder_id_str = str(folder["_id"])
# First, verify that no other outstanding jobs are running on this dataset
existing_jobs = Job().findOne({
JOBCONST_DATASET_ID: folder_id_str,
'status': {
# Find jobs that are inactive, queued, or running
# https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py
'$in': [0, 1, 2]
},
})
if existing_jobs is not None:
raise RestException(
(f"A pipeline for {folder_id_str} is already running. "
"Only one outstanding job may be run at a time for "
"a dataset."))
user = self.getCurrentUser()
token = Token().createToken(user=user, days=14)
move_existing_result_to_auxiliary_folder(folder, user)
params: PipelineJob = {
"input_folder": folder_id_str,
"input_type": folder["meta"]["type"],
"output_folder": folder_id_str,
"pipeline": pipeline,
}
newjob = run_pipeline.apply_async(
queue="pipelines",
kwargs=dict(
params=params,
girder_job_title=
f"Running {pipeline['name']} on {str(folder['name'])}",
girder_client_token=str(token["_id"]),
girder_job_type="pipelines",
),
)
newjob.job[JOBCONST_DATASET_ID] = folder_id_str
newjob.job[JOBCONST_RESULTS_FOLDER_ID] = folder_id_str
newjob.job[JOBCONST_PIPELINE_NAME] = pipeline['name']
# Allow any users with accecss to the input data to also
# see and possibly manage the job
Job().copyAccessPolicies(folder, newjob.job)
Job().save(newjob.job)
return newjob.job
def updateStep(self, slurmJobId):
job = Job().findOne(
{'otherFields.slurm_info.slurm_id': int(slurmJobId)})
# send log to girder periodic
log_file_name = 'slurm-{}.{}.out'.format(
job['otherFields']['slurm_info']['name'], slurmJobId)
log_file_path = os.path.join(self._shared_partition_log, log_file_name)
f = open(log_file_path, "r")
content = f.read()
job['log'].append(content)
f.close()
job.save()
return 'Updated log'
def _handleUpload(event):
upload, file = event.info['upload'], event.info['file']
try:
reference = json.loads(upload.get('reference'))
except (TypeError, ValueError):
return
if 'photomorph' in reference and 'photomorphOrdinal' in reference:
# TODO this is insecure. Should access check the item.
item = Item().load(file['itemId'], force=True, exc=True)
item['originalName'] = item['name']
name = '%05d_%s' % (reference['photomorphOrdinal'], item['name'])
item['name'] = name
item['photomorphTakenDate'] = _extractDate(file)
Item().save(item)
file['name'] = name
File().save(file)
try:
createThumbnail(width=128,
height=128,
crop=True,
fileId=file['_id'],
attachToType='item',
attachToId=item['_id'])
except Exception:
logger.exception('Failure during photomorph thumbnailing')
elif reference.get('photomorph') and 'resultType' in reference:
Folder().update({'_id': ObjectId(reference['folderId'])}, {
'$push': {
'photomorphOutputItems.%s' % reference['resultType']: {
'fileId': file['_id'],
'name': file['name']
}
}
},
multi=False)
elif reference.get('inpaintedImage'):
folder = Folder().load(reference['folderId'],
user=getCurrentUser(),
level=AccessType.WRITE)
if 'inpaintingJobId' in folder:
job = Job().load(folder['inpaintingJobId'], force=True)
job['inpaintingImageResultId'] = file['_id']
Job().save(job)
def runSlicerCliTasksDescriptionForFolder(self, folder, image, args, pullImage, params):
jobModel = Job()
token = Token().createToken(
days=3, scope='item_task.set_task_spec.%s' % folder['_id'], user=self.getCurrentUser())
job = jobModel.createJob(
title='Read docker task specs: %s' % image, type='folder.item_task_slicer_cli_description',
handler='worker_handler', user=self.getCurrentUser())
if args[-1:] == ['--xml']:
args = args[:-1]
jobOptions = {
'itemTaskId': folder['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': args + ['--xml'],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode': 'http',
'method': 'POST',
'format': 'text',
'url': '/'.join((utils.getWorkerApiUrl(), 'folder', str(folder['_id']),
'item_task_slicer_cli_xml')),
'headers': {'Girder-Token': token['_id']},
'params': {
'image': image,
'args': json.dumps(args),
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def executeTask(self, item, jobTitle, includeJobInfo, inputs, outputs):
user = self.getCurrentUser()
if jobTitle is None:
jobTitle = item['name']
task, handler = self._validateTask(item)
if task.get('mode') == 'girder_worker':
return runCeleryTask(item['meta']['itemTaskImport'], inputs)
jobModel = Job()
job = jobModel.createJob(
title=jobTitle, type='item_task', handler=handler, user=user)
# If this is a user auth token, we make an IO-enabled token
token = self.getCurrentToken()
tokenModel = Token()
if tokenModel.hasScope(token, TokenScope.USER_AUTH):
token = tokenModel.createToken(
user=user, days=7, scope=(TokenScope.DATA_READ, TokenScope.DATA_WRITE))
job['itemTaskTempToken'] = token['_id']
token = tokenModel.addScope(token, 'item_tasks.job_write:%s' % job['_id'])
job.update({
'itemTaskId': item['_id'],
'itemTaskBindings': {
'inputs': inputs,
'outputs': outputs
},
'kwargs': {
'task': task,
'inputs': self._transformInputs(inputs, token),
'outputs': self._transformOutputs(outputs, token, job, task, item['_id']),
'validate': False,
'auto_convert': False,
'cleanup': True
}
})
if includeJobInfo:
job['kwargs']['jobInfo'] = utils.jobInfoSpec(job)
if 'itemTaskCeleryQueue' in item.get('meta', {}):
job['celeryQueue'] = item['meta']['itemTaskCeleryQueue']
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
def delete(self, item, skipFileIds=None):
deleted = False
if 'largeImage' in item:
job = None
if 'jobId' in item['largeImage']:
try:
job = Job().load(item['largeImage']['jobId'],
force=True,
exc=True)
except ValidationException:
# The job has been deleted, but we still need to clean up
# the rest of the tile information
pass
if (item['largeImage'].get('expected') and job
and job.get('status')
in (JobStatus.QUEUED, JobStatus.RUNNING)):
# cannot cleanly remove the large image, since a conversion
# job is currently in progress
# TODO: cancel the job
# TODO: return a failure error code
return False
# If this file was created by the worker job, delete it
if 'jobId' in item['largeImage']:
# To eliminate all traces of the job, add
# if job:
# Job().remove(job)
del item['largeImage']['jobId']
if 'originalId' in item['largeImage']:
# The large image file should not be the original file
assert item['largeImage']['originalId'] != \
item['largeImage'].get('fileId')
if ('fileId' in item['largeImage'] and
(not skipFileIds
or item['largeImage']['fileId'] not in skipFileIds)):
file = File().load(id=item['largeImage']['fileId'],
force=True)
if file:
File().remove(file)
del item['largeImage']['originalId']
del item['largeImage']
item = self.save(item)
deleted = True
self.removeThumbnailFiles(item)
return deleted
def listInpaintingJobs(self, user, limit, offset, sort):
user = user or self.getCurrentUser()
cursor = Job().find(
{
'userId': user['_id'],
'inpaintingImageId': {
'$exists': True
}
},
sort=sort)
return list(Job().filterResultsByPermission(cursor=cursor,
user=self.getCurrentUser(),
level=AccessType.READ,
limit=limit,
offset=offset))
def wait_for_status(user, job, status):
"""
Utility to wait for a job model to move into a particular state.
:param job: The job model to wait on
:param status: The state to wait for.
:returns: True if the job model moved into the requested state, False otherwise.
"""
retries = 0
jobModel = Job()
while retries < 10:
job = jobModel.load(job['_id'], user=user)
if job['status'] == status:
return True
return False
def runInpainting(self, image, mask, folder):
basename = os.path.splitext(image['name'])[0]
outPath = VolumePath(basename + '_result.jpg')
artifactPath = VolumePath('job_artifacts')
job = docker_run.delay(
'zachmullen/inpainting:latest',
container_args=[
GirderFileIdToVolume(image['_id']),
GirderFileIdToVolume(mask['_id']), outPath, '--artifacts-dir',
artifactPath, '--progress-pipe',
ProgressPipe()
],
girder_job_title='Inpainting: %s' % image['name'],
girder_result_hooks=[
GirderUploadVolumePathToFolder(outPath,
folder['_id'],
upload_kwargs={
'reference':
json.dumps({
'inpaintedImage':
True,
'folderId':
str(folder['_id']),
})
}),
# GirderUploadVolumePathJobArtifact(artifactPath)
]).job
folder['inpaintingJobId'] = job['_id']
Folder().save(folder)
job['inpaintingImageId'] = image['_id']
job['inpaintingMaskId'] = mask['_id']
job['inpaintingFolderId'] = folder['_id']
return Job().save(job)
def attachParentJob(event):
"""Attach parentJob before a model is saved."""
job = event.info
if job.get('celeryParentTaskId'):
celeryParentTaskId = job['celeryParentTaskId']
parentJob = Job().findOne({'celeryTaskId': celeryParentTaskId})
event.info['parentId'] = parentJob['_id']
def jobInfoSpec(job, token=None, logPrint=True):
"""
Build the jobInfo specification for a task to write status and log output
back to a Girder job.
:param job: The job document representing the worker task.
:type job: dict
:param token: The token to use. Creates a job token if not passed.
:type token: str or dict
:param logPrint: Whether standard output from the job should be
"""
if token is None:
token = Job().createJobToken(job)
if isinstance(token, dict):
token = token['_id']
return {
'method': 'PUT',
'url': '/'.join((getWorkerApiUrl(), 'job', str(job['_id']))),
'reference': str(job['_id']),
'headers': {
'Girder-Token': token
},
'logPrint': logPrint
}
def execute_simulation(self, name, config, folder=None):
target_time = config.get('simulation', {}).get('run_time', 96)
user, token = self.getCurrentUser(returnToken=True)
folder_model: Folder = Folder()
job_model: Job = Job()
if folder is None:
folder = folder_model.findOne({
'parentId': user['_id'],
'name': 'Public',
'parentCollection': 'user'
})
if folder is None:
raise RestException(
'Could not find the user\'s "public" folder.')
job, simulation = simulation_runner(
config=config,
parent_folder=folder,
job_model=job_model,
run_name=name,
target_time=target_time,
token=token,
user=user,
)
return job
def _createThumbnails(server, admin, spec, cancel=False):
params = {'spec': json.dumps(spec)}
if cancel:
params['logInterval'] = 0
resp = server.request(method='PUT',
path='/large_image/thumbnails',
user=admin,
params=params)
assert utilities.respStatus(resp) == 200
job = resp.json
if cancel:
job = _waitForJobToBeRunning(job)
job = Job().cancelJob(job)
starttime = time.time()
while True:
assert time.time() - starttime < 30
resp = server.request('/job/%s' % str(job['_id']), user=admin)
assert utilities.respStatus(resp) == 200
if resp.json.get('status') == JobStatus.SUCCESS:
return True
if resp.json.get('status') == JobStatus.ERROR:
return False
if resp.json.get('status') == JobStatus.CANCELED:
return 'canceled'
time.sleep(0.1)
def cancel(event):
"""
This is bound to the "jobs.cancel" event, and will be triggered any time
a job is canceled. This handler will process any job that has the
handler field set to "worker_handler".
"""
job = event.info
if job['handler'] in ['worker_handler', 'celery_handler']:
# Stop event propagation and prevent default, we are using a custom state
event.stopPropagation().preventDefault()
celeryTaskId = job.get('celeryTaskId')
if celeryTaskId is None:
msg = (
"Unable to cancel Celery task. Job '%s' doesn't have a Celery task id."
% job['_id'])
logger.warn(msg)
return
if job['status'] not in [
CustomJobStatus.CANCELING, JobStatus.CANCELED,
JobStatus.SUCCESS, JobStatus.ERROR
]:
# Set the job status to canceling
Job().updateJob(job, status=CustomJobStatus.CANCELING)
# Send the revoke request.
asyncResult = AsyncResult(celeryTaskId, app=getCeleryApp())
asyncResult.revoke()
def setUp(self):
base.TestCase.setUp(self)
self.users = [User().createUser(
'usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n)
for n in range(3)]
self.jobModel = Job()
def importCollection(self, params):
user = self.getCurrentUser()
token = self.getCurrentToken()
tcga = self.getTCGACollection(level=AccessType.WRITE)
job = Job().createLocalJob(module='histomicsui.rest.tcga',
function='import_recursive',
kwargs={
'root': tcga,
'token': token
},
title='Import TCGA items',
user=user,
type='tcga_import_recursive',
public=False,
asynchronous=True)
Job().scheduleJob(job)
return job
def run(job):
jobModel = Job()
jobModel.updateJob(job, status=JobStatus.RUNNING)
try:
newFile = createThumbnail(**job['kwargs'])
log = 'Created thumbnail file %s.' % newFile['_id']
jobModel.updateJob(job, status=JobStatus.SUCCESS, log=log)
except Exception:
t, val, tb = sys.exc_info()
log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb))
jobModel.updateJob(job, status=JobStatus.ERROR, log=log)
raise
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
the user and challenge/phase administrators accordingly. Here, an
administrator is defined to be a user with WRITE access or above.
"""
isErrorStatus = False
try:
isErrorStatus = int(event.info['params'].get('status')) == JobStatus.ERROR
except (ValueError, TypeError):
pass
if (event.info['job']['type'] == 'covalic_score' and isErrorStatus):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
# Create minimal log that contains only Covalic errors.
# Use full log if no Covalic-specific errors are found.
# Fetch log from model, because log in event may not be up-to-date.
job = Job().load(
event.info['job']['_id'], includeLog=True, force=True)
log = job.get('log')
minimalLog = None
if log:
log = ''.join(log)
minimalLog = '\n'.join([line[len(JOB_LOG_PREFIX):].strip()
for line in log.splitlines()
if line.startswith(JOB_LOG_PREFIX)])
if not minimalLog:
minimalLog = log
submission = Submission().load(
event.info['job']['covalicSubmissionId'])
phase = Phase().load(
submission['phaseId'], force=True)
challenge = Challenge().load(
phase['challengeId'], force=True)
user = User().load(
event.info['job']['userId'], force=True)
rescoring = job.get('rescoring', False)
# Mail admins, include full log
emails = sorted(getPhaseUserEmails(
phase, AccessType.WRITE, includeChallengeUsers=True))
html = mail_utils.renderTemplate('covalic.submissionErrorAdmin.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'user': user,
'host': covalicHost,
'log': log
})
mail_utils.sendEmail(
to=emails, subject='Submission processing error', text=html)
# Mail user, include minimal log
if not rescoring:
html = mail_utils.renderTemplate('covalic.submissionErrorUser.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'host': covalicHost,
'log': minimalLog
})
mail_utils.sendEmail(
to=user['email'], subject='Submission processing error', text=html)
def runJsonTasksDescriptionForItem(self, item, image, taskName, setName, setDescription,
pullImage, params):
if 'meta' not in item:
item['meta'] = {}
if image is None:
image = item.get('meta', {}).get('itemTaskSpec', {}).get('docker_image')
if not image:
raise RestException(
'You must pass an image parameter, or set the itemTaskSpec.docker_image '
'field of the item.')
jobModel = Job()
token = Token().createToken(
days=3, scope='item_task.set_task_spec.%s' % item['_id'],
user=self.getCurrentUser())
job = jobModel.createJob(
title='Read docker task specs: %s' % image, type='item.item_task_json_description',
handler='worker_handler', user=self.getCurrentUser())
jobOptions = {
'itemTaskId': item['_id'],
'kwargs': {
'task': {
'mode': 'docker',
'docker_image': image,
'container_args': [],
'pull_image': pullImage,
'outputs': [{
'id': '_stdout',
'format': 'text'
}],
},
'outputs': {
'_stdout': {
'mode': 'http',
'method': 'PUT',
'format': 'text',
'url': '/'.join((utils.getWorkerApiUrl(), 'item', str(item['_id']),
'item_task_json_specs')),
'headers': {'Girder-Token': token['_id']},
'params': {
'image': image,
'taskName': taskName,
'setName': setName,
'setDescription': setDescription,
'pullImage': pullImage
}
}
},
'jobInfo': utils.jobInfoSpec(job),
'validate': False,
'auto_convert': False
}
}
job.update(jobOptions)
job = jobModel.save(job)
jobModel.scheduleJob(job)
return job
class JobsTestCase(base.TestCase):
def setUp(self):
base.TestCase.setUp(self)
self.users = [User().createUser(
'usr' + str(n), 'passwd', 'tst', 'usr', '*****@*****.**' % n)
for n in range(3)]
self.jobModel = Job()
def testJobs(self):
self.job = None
def schedule(event):
self.job = event.info
if self.job['handler'] == 'my_handler':
self.job['status'] = JobStatus.RUNNING
self.job = self.jobModel.save(self.job)
self.assertEqual(self.job['args'], ('hello', 'world'))
self.assertEqual(self.job['kwargs'], {'a': 'b'})
events.bind('jobs.schedule', 'test', schedule)
# Create a job
job = self.jobModel.createJob(
title='Job Title', type='my_type', args=('hello', 'world'),
kwargs={'a': 'b'}, user=self.users[1], handler='my_handler',
public=False)
self.assertEqual(self.job, None)
self.assertEqual(job['status'], JobStatus.INACTIVE)
# Schedule the job, make sure our handler was invoked
self.jobModel.scheduleJob(job)
self.assertEqual(self.job['_id'], job['_id'])
self.assertEqual(self.job['status'], JobStatus.RUNNING)
# Since the job is not public, user 2 should not have access
path = '/job/%s' % job['_id']
resp = self.request(path, user=self.users[2])
self.assertStatus(resp, 403)
resp = self.request(path, user=self.users[2], method='PUT')
self.assertStatus(resp, 403)
resp = self.request(path, user=self.users[2], method='DELETE')
self.assertStatus(resp, 403)
# If no user is specified, we should get a 401 error
resp = self.request(path, user=None)
self.assertStatus(resp, 401)
# Make sure user who created the job can see it
resp = self.request(path, user=self.users[1])
self.assertStatusOk(resp)
# We should be able to update the job as the user who created it
resp = self.request(path, method='PUT', user=self.users[1], params={
'log': 'My log message\n'
})
self.assertStatusOk(resp)
# We should be able to create a job token and use that to update it too
token = self.jobModel.createJobToken(job)
resp = self.request(path, method='PUT', params={
'log': 'append message',
'token': token['_id']
})
self.assertStatusOk(resp)
# We shouldn't get the log back in this case
self.assertNotIn('log', resp.json)
# Do a fetch on the job itself to get the log
resp = self.request(path, user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(
resp.json['log'], ['My log message\n', 'append message'])
# Test overwriting the log and updating status
resp = self.request(path, method='PUT', params={
'log': 'overwritten log',
'overwrite': 'true',
'status': JobStatus.SUCCESS,
'token': token['_id']
})
self.assertStatusOk(resp)
self.assertNotIn('log', resp.json)
self.assertEqual(resp.json['status'], JobStatus.SUCCESS)
job = self.jobModel.load(job['_id'], force=True, includeLog=True)
self.assertEqual(job['log'], ['overwritten log'])
# We should be able to delete the job as the user who created it
resp = self.request(path, user=self.users[1], method='DELETE')
self.assertStatusOk(resp)
job = self.jobModel.load(job['_id'], force=True)
self.assertIsNone(job)
def testLegacyLogBehavior(self):
# Force save a job with a string log to simulate a legacy job record
job = self.jobModel.createJob(
title='legacy', type='legacy', user=self.users[1], save=False)
job['log'] = 'legacy log'
job = self.jobModel.save(job, validate=False)
self.assertEqual(job['log'], 'legacy log')
# Load the record, we should now get the log as a list
job = self.jobModel.load(job['_id'], force=True, includeLog=True)
self.assertEqual(job['log'], ['legacy log'])
def testListJobs(self):
job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=False)
anonJob = self.jobModel.createJob(title='Anon job', type='t')
# Ensure timestamp for public job is strictly higher (ms resolution)
time.sleep(0.1)
publicJob = self.jobModel.createJob(
title='Anon job', type='t', public=True)
# User 1 should be able to see their own jobs
resp = self.request('/job', user=self.users[1], params={
'userId': self.users[1]['_id']
})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(resp.json[0]['_id'], str(job['_id']))
# User 2 should not see user 1's jobs in the list
resp = self.request('/job', user=self.users[2], params={
'userId': self.users[1]['_id']
})
self.assertEqual(resp.json, [])
# Omitting a userId should assume current user
resp = self.request('/job', user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(resp.json[0]['_id'], str(job['_id']))
# Explicitly passing "None" should show anonymous jobs
resp = self.request('/job', user=self.users[0], params={
'userId': 'none'
})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 2)
self.assertEqual(resp.json[0]['_id'], str(publicJob['_id']))
self.assertEqual(resp.json[1]['_id'], str(anonJob['_id']))
# Non-admins should only see public anon jobs
resp = self.request('/job', params={'userId': 'none'})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(resp.json[0]['_id'], str(publicJob['_id']))
def testListAllJobs(self):
self.jobModel.createJob(title='user 0 job', type='t', user=self.users[0], public=False)
self.jobModel.createJob(title='user 1 job', type='t', user=self.users[1], public=False)
self.jobModel.createJob(title='user 1 job', type='t', user=self.users[1], public=True)
self.jobModel.createJob(title='user 2 job', type='t', user=self.users[2])
self.jobModel.createJob(title='anonymous job', type='t')
self.jobModel.createJob(title='anonymous public job', type='t2', public=True)
# User 0, as a site admin, should be able to see all jobs
resp = self.request('/job/all', user=self.users[0])
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 6)
# get with filter
resp = self.request('/job/all', user=self.users[0], params={
'types': json.dumps(['t']),
'statuses': json.dumps([0])
})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 5)
# get with unmet filter conditions
resp = self.request('/job/all', user=self.users[0], params={
'types': json.dumps(['nonexisttype'])
})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 0)
# User 1, as non site admin, should encounter http 403 (Forbidden)
resp = self.request('/job/all', user=self.users[1])
self.assertStatus(resp, 403)
# Not authenticated user should encounter http 401 (unauthorized)
resp = self.request('/job/all')
self.assertStatus(resp, 401)
def testFiltering(self):
job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=True)
job['_some_other_field'] = 'foo'
job = self.jobModel.save(job)
resp = self.request('/job/%s' % job['_id'])
self.assertStatusOk(resp)
self.assertTrue('created' in resp.json)
self.assertTrue('_some_other_field' not in resp.json)
self.assertTrue('kwargs' not in resp.json)
self.assertTrue('args' not in resp.json)
resp = self.request('/job/%s' % job['_id'], user=self.users[0])
self.assertTrue('kwargs' in resp.json)
self.assertTrue('args' in resp.json)
self.jobModel.exposeFields(level=AccessType.READ, fields={'_some_other_field'})
self.jobModel.hideFields(level=AccessType.READ, fields={'created'})
resp = self.request('/job/%s' % job['_id'])
self.assertStatusOk(resp)
self.assertEqual(resp.json['_some_other_field'], 'foo')
self.assertTrue('created' not in resp.json)
def testJobProgressAndNotifications(self):
job = self.jobModel.createJob(title='a job', type='t', user=self.users[1], public=True)
path = '/job/%s' % job['_id']
resp = self.request(path)
self.assertEqual(resp.json['progress'], None)
self.assertEqual(resp.json['timestamps'], [])
resp = self.request(path, method='PUT', user=self.users[1], params={
'progressTotal': 100,
'progressCurrent': 3,
'progressMessage': 'Started',
'notify': 'false',
'status': JobStatus.QUEUED
})
self.assertStatusOk(resp)
self.assertEqual(resp.json['progress'], {
'total': 100,
'current': 3,
'message': 'Started',
'notificationId': None
})
# The status update should make it so we now have a timestamp
self.assertEqual(len(resp.json['timestamps']), 1)
self.assertEqual(resp.json['timestamps'][0]['status'], JobStatus.QUEUED)
self.assertIn('time', resp.json['timestamps'][0])
# If the status does not change on update, no timestamp should be added
resp = self.request(path, method='PUT', user=self.users[1], params={
'status': JobStatus.QUEUED
})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json['timestamps']), 1)
self.assertEqual(resp.json['timestamps'][0]['status'], JobStatus.QUEUED)
# We passed notify=false, so we should only have the job creation notification
resp = self.request(path='/notification/stream', method='GET',
user=self.users[1], isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
# Update progress with notify=true (the default)
resp = self.request(path, method='PUT', user=self.users[1], params={
'progressCurrent': 50,
'progressMessage': 'Something bad happened',
'status': JobStatus.ERROR
})
self.assertStatusOk(resp)
self.assertNotEqual(resp.json['progress']['notificationId'], None)
# We should now see three notifications (job created + job status + progress)
resp = self.request(path='/notification/stream', method='GET',
user=self.users[1], isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
job = self.jobModel.load(job['_id'], force=True)
self.assertEqual(len(messages), 3)
creationNotify = messages[0]
progressNotify = messages[1]
statusNotify = messages[2]
self.assertEqual(creationNotify['type'], 'job_created')
self.assertEqual(creationNotify['data']['_id'], str(job['_id']))
self.assertEqual(statusNotify['type'], 'job_status')
self.assertEqual(statusNotify['data']['_id'], str(job['_id']))
self.assertEqual(int(statusNotify['data']['status']), JobStatus.ERROR)
self.assertNotIn('kwargs', statusNotify['data'])
self.assertNotIn('log', statusNotify['data'])
self.assertEqual(progressNotify['type'], 'progress')
self.assertEqual(progressNotify['data']['title'], job['title'])
self.assertEqual(progressNotify['data']['current'], float(50))
self.assertEqual(progressNotify['data']['state'], 'error')
self.assertEqual(progressNotify['_id'], str(job['progress']['notificationId']))
def testDotsInKwargs(self):
kwargs = {
'$key.with.dots': 'value',
'foo': [{
'moar.dots': True
}]
}
job = self.jobModel.createJob(title='dots', type='x', user=self.users[0], kwargs=kwargs)
# Make sure we can update a job and notification creation works
self.jobModel.updateJob(job, status=JobStatus.QUEUED, notify=True)
self.assertEqual(job['kwargs'], kwargs)
resp = self.request('/job/%s' % job['_id'], user=self.users[0])
self.assertStatusOk(resp)
self.assertEqual(resp.json['kwargs'], kwargs)
job = self.jobModel.load(job['_id'], force=True)
self.assertEqual(job['kwargs'], kwargs)
job = self.jobModel.filter(job, self.users[0])
self.assertEqual(job['kwargs'], kwargs)
job = self.jobModel.filter(job, self.users[1])
self.assertFalse('kwargs' in job)
def testLocalJob(self):
job = self.jobModel.createLocalJob(
title='local', type='local', user=self.users[0], kwargs={
'hello': 'world'
}, module='plugin_tests.local_job_impl')
self.jobModel.scheduleJob(job)
job = self.jobModel.load(job['_id'], force=True, includeLog=True)
self.assertEqual(job['log'], ['job ran!'])
job = self.jobModel.createLocalJob(
title='local', type='local', user=self.users[0], kwargs={
'hello': 'world'
}, module='plugin_tests.local_job_impl', function='fail')
self.jobModel.scheduleJob(job)
job = self.jobModel.load(job['_id'], force=True, includeLog=True)
self.assertEqual(job['log'], ['job failed'])
def testValidateCustomStatus(self):
job = self.jobModel.createJob(title='test', type='x', user=self.users[0])
def validateStatus(event):
if event.info == 1234:
event.preventDefault().addResponse(True)
def validTransitions(event):
if event.info['status'] == 1234:
event.preventDefault().addResponse([JobStatus.INACTIVE])
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status=1234) # Should fail
with events.bound('jobs.status.validate', 'test', validateStatus), \
events.bound('jobs.status.validTransitions', 'test', validTransitions):
self.jobModel.updateJob(job, status=1234) # Should work
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status=4321) # Should fail
def testValidateCustomStrStatus(self):
job = self.jobModel.createJob(title='test', type='x', user=self.users[0])
def validateStatus(event):
states = ['a', 'b', 'c']
if event.info in states:
event.preventDefault().addResponse(True)
def validTransitions(event):
if event.info['status'] == 'a':
event.preventDefault().addResponse([JobStatus.INACTIVE])
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status='a')
with events.bound('jobs.status.validate', 'test', validateStatus), \
events.bound('jobs.status.validTransitions', 'test', validTransitions):
self.jobModel.updateJob(job, status='a')
self.assertEqual(job['status'], 'a')
with self.assertRaises(ValidationException), \
events.bound('jobs.status.validate', 'test', validateStatus):
self.jobModel.updateJob(job, status='foo')
def testUpdateOtherFields(self):
job = self.jobModel.createJob(title='test', type='x', user=self.users[0])
job = self.jobModel.updateJob(job, otherFields={'other': 'fields'})
self.assertEqual(job['other'], 'fields')
def testCancelJob(self):
job = self.jobModel.createJob(title='test', type='x', user=self.users[0])
# add to the log
job = self.jobModel.updateJob(job, log='entry 1\n')
# Reload without the log
job = self.jobModel.load(id=job['_id'], force=True)
self.assertEqual(len(job.get('log', [])), 0)
# Cancel
job = self.jobModel.cancelJob(job)
self.assertEqual(job['status'], JobStatus.CANCELED)
# Reloading should still have the log and be canceled
job = self.jobModel.load(id=job['_id'], force=True, includeLog=True)
self.assertEqual(job['status'], JobStatus.CANCELED)
self.assertEqual(len(job.get('log', [])), 1)
def testCancelJobEndpoint(self):
job = self.jobModel.createJob(title='test', type='x', user=self.users[0])
# Ensure requires write perms
jobCancelUrl = '/job/%s/cancel' % job['_id']
resp = self.request(jobCancelUrl, user=self.users[1], method='PUT')
self.assertStatus(resp, 403)
# Try again with the right user
jobCancelUrl = '/job/%s/cancel' % job['_id']
resp = self.request(jobCancelUrl, user=self.users[0], method='PUT')
self.assertStatusOk(resp)
self.assertEqual(resp.json['status'], JobStatus.CANCELED)
def testJobsTypesAndStatuses(self):
self.jobModel.createJob(title='user 0 job', type='t1', user=self.users[0], public=False)
self.jobModel.createJob(title='user 1 job', type='t2', user=self.users[1], public=False)
self.jobModel.createJob(title='user 1 job', type='t3', user=self.users[1], public=True)
self.jobModel.createJob(title='user 2 job', type='t4', user=self.users[2])
self.jobModel.createJob(title='anonymous job', type='t5')
self.jobModel.createJob(title='anonymous public job', type='t6', public=True)
# User 1, as non site admin, should encounter http 403 (Forbidden)
resp = self.request('/job/typeandstatus/all', user=self.users[1])
self.assertStatus(resp, 403)
# Admin user gets all types and statuses
resp = self.request('/job/typeandstatus/all', user=self.users[0])
self.assertStatusOk(resp)
self.assertEqual(len(resp.json['types']), 6)
self.assertEqual(len(resp.json['statuses']), 1)
# standard user gets types and statuses of its own jobs
resp = self.request('/job/typeandstatus', user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(len(resp.json['types']), 2)
self.assertEqual(len(resp.json['statuses']), 1)
def testDefaultParentId(self):
job = self.jobModel.createJob(title='Job', type='Job', user=self.users[0])
# If not specified parentId should be None
self.assertEquals(job['parentId'], None)
def testIsParentIdCorrect(self):
parentJob = self.jobModel.createJob(
title='Parent Job', type='Parent Job', user=self.users[0])
childJob = self.jobModel.createJob(
title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
# During initialization parent job should be set correctly
self.assertEqual(childJob['parentId'], parentJob['_id'])
def testSetParentCorrectly(self):
parentJob = self.jobModel.createJob(
title='Parent Job', type='Parent Job', user=self.users[0])
childJob = self.jobModel.createJob(title='Child Job', type='Child Job', user=self.users[0])
self.jobModel.setParentJob(childJob, parentJob)
# After setParentJob method is called parent job should be set correctly
self.assertEqual(childJob['parentId'], parentJob['_id'])
def testParentCannotBeEqualToChild(self):
childJob = self.jobModel.createJob(title='Child Job', type='Child Job', user=self.users[0])
# Cannot set a job as it's own parent
with self.assertRaises(ValidationException):
self.jobModel.setParentJob(childJob, childJob)
def testParentIdCannotBeOverridden(self):
parentJob = self.jobModel.createJob(
title='Parent Job', type='Parent Job', user=self.users[0])
anotherParentJob = self.jobModel.createJob(
title='Another Parent Job', type='Parent Job', user=self.users[0])
childJob = self.jobModel.createJob(
title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
with self.assertRaises(ValidationException):
# If parent job is set, cannot be overridden
self.jobModel.setParentJob(childJob, anotherParentJob)
def testListChildJobs(self):
parentJob = self.jobModel.createJob(
title='Parent Job', type='Parent Job', user=self.users[0])
childJob = self.jobModel.createJob(
title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
self.jobModel.createJob(
title='Another Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
# Should return a list with 2 jobs
self.assertEquals(len(list(self.jobModel.listChildJobs(parentJob))), 2)
# Should return an empty list
self.assertEquals(len(list(self.jobModel.listChildJobs(childJob))), 0)
def testListChildJobsRest(self):
parentJob = self.jobModel.createJob(
title='Parent Job', type='Parent Job', user=self.users[0])
childJob = self.jobModel.createJob(
title='Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
self.jobModel.createJob(
title='Another Child Job', type='Child Job', user=self.users[0], parentJob=parentJob)
resp = self.request('/job', user=self.users[0],
params={'parentId': str(parentJob['_id'])})
resp2 = self.request('/job', user=self.users[0],
params={'parentId': str(childJob['_id'])})
self.assertStatusOk(resp)
self.assertStatusOk(resp2)
# Should return a list with 2 jobs
self.assertEquals(len(resp.json), 2)
# Should return an empty list
self.assertEquals(len(resp2.json), 0)
def testCreateJobRest(self):
resp = self.request('/job', method='POST',
user=self.users[0],
params={'title': 'job', 'type': 'job'})
# If user does not have the necessary token status is 403
self.assertStatus(resp, 403)
token = Token().createToken(scope=REST_CREATE_JOB_TOKEN_SCOPE)
resp2 = self.request(
'/job', method='POST', token=token, params={'title': 'job', 'type': 'job'})
# If user has the necessary token status is 200
self.assertStatusOk(resp2)
def testJobStateTransitions(self):
job = self.jobModel.createJob(
title='user 0 job', type='t1', user=self.users[0], public=False)
# We can't move straight to SUCCESS
with self.assertRaises(ValidationException):
job = self.jobModel.updateJob(job, status=JobStatus.SUCCESS)
self.jobModel.updateJob(job, status=JobStatus.QUEUED)
self.jobModel.updateJob(job, status=JobStatus.RUNNING)
self.jobModel.updateJob(job, status=JobStatus.ERROR)
# We shouldn't be able to move backwards
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status=JobStatus.QUEUED)
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status=JobStatus.RUNNING)
with self.assertRaises(ValidationException):
self.jobModel.updateJob(job, status=JobStatus.INACTIVE)
def testJobSaveEventModification(self):
def customSave(event):
kwargs = json_util.loads(event.info['kwargs'])
kwargs['key2'] = 'newvalue'
event.info['kwargs'] = json_util.dumps(kwargs)
job = self.jobModel.createJob(title='A job', type='t', user=self.users[1], public=True)
job['kwargs'] = {'key1': 'value1', 'key2': 'value2'}
with events.bound('model.job.save', 'test', customSave):
job = self.jobModel.save(job)
self.assertEqual(job['kwargs']['key2'], 'newvalue')
def testConfigureItemTaskFromSlicerCli(self):
# Create a new item that will become a task
item = Item().createItem(name='placeholder', creator=self.admin, folder=self.privateFolder)
# Create task to introspect container
with mock.patch('girder_jobs.models.job.Job.scheduleJob') as scheduleMock:
resp = self.request(
'/item/%s/item_task_slicer_cli_description' % item['_id'], method='POST', params={
'image': 'johndoe/foo:v5',
'args': json.dumps(['--foo', 'bar'])
}, user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json['_modelType'], 'job')
self.assertEqual(len(scheduleMock.mock_calls), 1)
job = scheduleMock.mock_calls[0][1][0]
self.assertEqual(job['handler'], 'worker_handler')
self.assertEqual(job['itemTaskId'], item['_id'])
self.assertEqual(job['kwargs']['outputs']['_stdout']['method'], 'PUT')
self.assertTrue(job['kwargs']['outputs']['_stdout']['url'].endswith(
'item/%s/item_task_slicer_cli_xml' % item['_id']))
token = job['kwargs']['outputs']['_stdout']['headers']['Girder-Token']
# Task should not be registered until we get the callback
resp = self.request('/item_task', user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
# Image and args should be stored in the item metadata
item = Item().load(item['_id'], force=True)
self.assertEqual(item['meta']['itemTaskSpec']['docker_image'], 'johndoe/foo:v5')
self.assertEqual(item['meta']['itemTaskSlicerCliArgs'], ['--foo', 'bar'])
# Simulate callback from introspection job
with open(os.path.join(os.path.dirname(__file__), 'slicer_cli.xml')) as f:
xml = f.read()
resp = self.request(
'/item/%s/item_task_slicer_cli_xml' % item['_id'], method='PUT', params={
'setName': True,
'setDescription': True
}, token=token, body=xml, type='application/xml')
self.assertStatusOk(resp)
# We should only be able to see tasks we have read access on
resp = self.request('/item_task')
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
resp = self.request('/item_task', user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(resp.json[0]['_id'], str(item['_id']))
item = Item().load(item['_id'], force=True)
self.assertEqual(item['name'], 'PET phantom detector CLI')
self.assertEqual(
item['description'],
u'**Description**: Detects positions of PET/CT pocket phantoms in PET image.\n\n'
u'**Author(s)**: Girder Developers\n\n**Version**: 1.0\n\n'
u'**License**: Apache 2.0\n\n**Acknowledgements**: *none*\n\n'
u'*This description was auto-generated from the Slicer CLI XML specification.*'
)
self.assertTrue(item['meta']['isItemTask'])
self.assertEqual(item['meta']['itemTaskSpec'], {
'mode': 'docker',
'docker_image': 'johndoe/foo:v5',
'container_args': [
'--foo', 'bar', '--InputImage=$input{--InputImage}',
'--MaximumLineStraightnessDeviation=$input{--MaximumLineStraightnessDeviation}',
'--MaximumRadius=$input{--MaximumRadius}',
'--MaximumSphereDistance=$input{--MaximumSphereDistance}',
'--MinimumRadius=$input{--MinimumRadius}',
'--MinimumSphereActivity=$input{--MinimumSphereActivity}',
'--MinimumSphereDistance=$input{--MinimumSphereDistance}',
'--SpheresPerPhantom=$input{--SpheresPerPhantom}', '$flag{--StrictSorting}',
'--DetectedPoints=$output{--DetectedPoints}'
],
'inputs': [{
'description': 'Input image to be analysed.',
'format': 'image',
'name': 'InputImage', 'type': 'image', 'id': '--InputImage',
'target': 'filepath'
}, {
'description': 'Used for eliminating detections which are not in a straight line. '
'Unit: multiples of geometric average of voxel spacing',
'format': 'number',
'default': {'data': 1.0},
'type': 'number',
'id': '--MaximumLineStraightnessDeviation',
'name': 'MaximumLineStraightnessDeviation'
}, {
'description': 'Used for eliminating too big blobs. Unit: millimeter [mm]',
'format': 'number', 'default': {'data': 20.0},
'type': 'number',
'id': '--MaximumRadius',
'name': 'MaximumRadius'
}, {
'description': 'Signifies maximum distance between adjacent sphere centers [mm]. '
'Used to separate phantoms from tumors.',
'format': 'number', 'default': {'data': 40.0},
'type': 'number',
'id': '--MaximumSphereDistance',
'name': 'MaximumSphereDistance'
}, {
'description': 'Used for eliminating too small blobs. Unit: millimeter [mm]',
'format': 'number',
'default': {'data': 3.0},
'type': 'number',
'id': '--MinimumRadius',
'name': 'MinimumRadius'
}, {
'description': 'Used for thresholding in blob detection. '
'Unit: becquerels per milliliter [Bq/ml]',
'format': 'number', 'default': {'data': 5000.0},
'type': 'number',
'id': '--MinimumSphereActivity',
'name': 'MinimumSphereActivity'
}, {
'description': 'Signifies minimum distance between adjacent sphere centers [mm]. '
'Used to separate phantoms from tumors.',
'format': 'number',
'default': {'data': 30.0},
'type': 'number',
'id': '--MinimumSphereDistance',
'name': 'MinimumSphereDistance'
}, {
'description': 'What kind of phantom are we working with here?',
'format': 'number-enumeration',
'default': {'data': 3},
'type': 'number-enumeration',
'id': '--SpheresPerPhantom',
'name': 'SpheresPerPhantom',
'values': [2, 3]
}, {
'description': 'Controls whether spheres within a phantom must have descending '
'activities. If OFF, they can have approximately same activities '
'(within 15%).',
'format': 'boolean',
'default': {'data': False},
'type': 'boolean',
'id': '--StrictSorting',
'name': 'StrictSorting'
}],
'outputs': [{
'description': 'Fiducial points, one for each detected sphere. '
'Will be multiple of 3.',
'format': 'new-file',
'name': 'DetectedPoints',
'type': 'new-file',
'id': '--DetectedPoints',
'target': 'filepath'
}]
})
# Shouldn't be able to run the task if we don't have execute permission flag
Folder().setUserAccess(
self.privateFolder, user=self.user, level=AccessType.READ, save=True)
resp = self.request(
'/item_task/%s/execution' % item['_id'], method='POST', user=self.user)
self.assertStatus(resp, 403)
# Grant the user permission, and run the task
Folder().setUserAccess(
self.privateFolder, user=self.user, level=AccessType.WRITE,
flags=ACCESS_FLAG_EXECUTE_TASK, currentUser=self.admin, save=True)
inputs = {
'--InputImage': {
'mode': 'girder',
'resource_type': 'item',
'id': str(item['_id'])
},
'--MaximumLineStraightnessDeviation': {
'mode': 'inline',
'data': 1
},
'--MaximumRadius': {
'mode': 'inline',
'data': 20
},
'--MaximumSphereDistance': {
'mode': 'inline',
'data': 40
},
'--MinimumRadius': {
'mode': 'inline',
'data': 3
},
'--MinimumSphereActivity': {
'mode': 'inline',
'data': 5000
},
'--MinimumSphereDistance': {
'mode': 'inline',
'data': 30
},
'--SpheresPerPhantom': {
'mode': 'inline',
'data': 3},
'--StrictSorting': {
'mode': 'inline',
'data': False
}
}
outputs = {
'--DetectedPoints': {
'mode': 'girder',
'parent_id': str(self.privateFolder['_id']),
'parent_type': 'folder',
'name': 'test.txt'
}
}
# Ensure task was scheduled
with mock.patch('girder_jobs.models.job.Job.scheduleJob') as scheduleMock:
resp = self.request(
'/item_task/%s/execution' % item['_id'], method='POST', user=self.user, params={
'inputs': json.dumps(inputs),
'outputs': json.dumps(outputs)
})
self.assertEqual(len(scheduleMock.mock_calls), 1)
self.assertStatusOk(resp)
job = resp.json
self.assertEqual(job['_modelType'], 'job')
self.assertNotIn('kwargs', job) # ordinary user can't see kwargs
jobModel = Job()
job = jobModel.load(job['_id'], force=True)
output = job['kwargs']['outputs']['--DetectedPoints']
# Simulate output from the worker
contents = b'Hello world'
resp = self.request(
path='/file', method='POST', token=output['token'], params={
'parentType': output['parent_type'],
'parentId': output['parent_id'],
'name': output['name'],
'size': len(contents),
'mimeType': 'text/plain',
'reference': output['reference']
})
self.assertStatusOk(resp)
uploadId = resp.json['_id']
fields = [('offset', 0), ('uploadId', uploadId)]
files = [('chunk', output['name'], contents)]
resp = self.multipartRequest(
path='/file/chunk', fields=fields, files=files, token=output['token'])
self.assertStatusOk(resp)
file = resp.json
self.assertEqual(file['_modelType'], 'file')
self.assertEqual(file['size'], 11)
self.assertEqual(file['mimeType'], 'text/plain')
file = File().load(file['_id'], force=True)
# Make sure temp token is removed once we change job status to final state
job = jobModel.load(job['_id'], force=True)
self.assertIn('itemTaskTempToken', job)
# Transition through states to SUCCESS
job = jobModel.updateJob(job, status=JobStatus.QUEUED)
job = jobModel.updateJob(job, status=JobStatus.RUNNING)
job = jobModel.updateJob(job, status=JobStatus.SUCCESS)
self.assertNotIn('itemTaskTempToken', job)
self.assertIn('itemTaskBindings', job)
# Wait for async data.process event to bind output provenance
start = time.time()
while time.time() - start < 15:
job = jobModel.load(job['_id'], force=True)
if 'itemId' in job['itemTaskBindings']['outputs']['--DetectedPoints']:
break
else:
time.sleep(0.2)
else:
raise Exception('Output binding did not occur in time')
self.assertEqual(
job['itemTaskBindings']['outputs']['--DetectedPoints']['itemId'], file['itemId'])
