def _createHistogramJob(self, **kwargs):
from girder.plugins.jobs.models.job import Job
from girder.plugins.jobs.constants import JobStatus
from girder.plugins.histogram.models.histogram import Histogram
file, item = self._uploadFile('plugins/large_image/plugin_tests/test_files/test_L_8.png')
token = Token().createToken(self.admin)
doc = Histogram().createHistogramJob(item, file, user=self.admin,
token=token, **kwargs)
complete = (JobStatus.SUCCESS, JobStatus.ERROR, JobStatus.CANCELED)
starttime = time.time()
while True:
self.assertTrue(time.time() - starttime < 30)
job = Job().load(doc['_id'], user=self.admin, exc=True)
if job.get('status') in complete:
break
time.sleep(0.1)
if job.get('log'):
print(job.get('log'))
assert job.get('status') == JobStatus.SUCCESS
return ObjectId(job['_id'])
def deleteIncompleteTiles(self, params):
result = {'removed': 0}
while True:
item = Item().findOne({'largeImage.expected': True})
if not item:
break
job = Job().load(item['largeImage']['jobId'], force=True)
if job and job.get('status') in (
JobStatus.QUEUED, JobStatus.RUNNING):
job = Job().cancelJob(job)
if job and job.get('status') in (
JobStatus.QUEUED, JobStatus.RUNNING):
result['message'] = ('The job for item %s could not be '
'canceled' % (str(item['_id'])))
break
ImageItem().delete(item)
result['removed'] += 1
return result
def delete(self, item, skipFileIds=None):
deleted = False
if 'largeImage' in item:
job = None
if 'jobId' in item['largeImage']:
try:
job = Job().load(item['largeImage']['jobId'],
force=True,
exc=True)
except ValidationException:
# The job has been deleted, but we still need to clean up
# the rest of the tile information
pass
if (item['largeImage'].get('expected') and job
and job.get('status')
in (JobStatus.QUEUED, JobStatus.RUNNING)):
# cannot cleanly remove the large image, since a conversion
# job is currently in progress
# TODO: cancel the job
# TODO: return a failure error code
return False
# If this file was created by the worker job, delete it
if 'jobId' in item['largeImage']:
if job:
# TODO: does this eliminate all traces of the job?
# TODO: do we want to remove the original job?
Job().remove(job)
del item['largeImage']['jobId']
if 'originalId' in item['largeImage']:
# The large image file should not be the original file
assert item['largeImage']['originalId'] != \
item['largeImage'].get('fileId')
if ('fileId' in item['largeImage'] and
(not skipFileIds
or item['largeImage']['fileId'] not in skipFileIds)):
file = File().load(id=item['largeImage']['fileId'],
force=True)
if file:
File().remove(file)
del item['largeImage']['originalId']
del item['largeImage']
item = self.save(item)
deleted = True
self.removeThumbnailFiles(item)
return deleted
def createThumbnailsJob(job):
"""
Create thumbnails for all of the large image items.
:param spec: an array, each entry of which is the parameter dictionary for
the model getThumbnail function.
"""
job = Job().updateJob(
job, log='Started creating large image thumbnails\n',
status=JobStatus.RUNNING)
checkedOrCreated = 0
failedImages = 0
try:
spec = job['kwargs']['spec']
logInterval = float(job['kwargs'].get('logInterval', 10))
for entry in spec:
job = Job().updateJob(
job, log='Creating thumbnails for %r\n' % entry)
lastLogTime = time.time()
items = Item().find({'largeImage.fileId': {'$exists': True}})
for item in items:
try:
ImageItem().getThumbnail(item, **entry)
checkedOrCreated += 1
except TileGeneralException as exc:
failedImages += 1
lastFailed = str(item['_id'])
logger.info('Failed to get thumbnail for item %s: %r' % (
lastFailed, exc))
# Periodically, log the state of the job and check if it was
# deleted or canceled.
if time.time() - lastLogTime > logInterval:
job = Job().updateJob(
job, log='Checked or created %d thumbnail file%s\n' % (
checkedOrCreated,
's' if checkedOrCreated != 1 else ''))
if failedImages:
job = Job().updateJob(
job, log='Failed on %d thumbnail file%s (last '
'failure on item %s)\n' % (
failedImages,
's' if failedImages != 1 else '', lastFailed))
lastLogTime = time.time()
# Check if the job was deleted or canceled; if so, quit
job = Job().load(id=job['_id'], force=True)
if (not job or job['status'] in (
JobStatus.CANCELED, JobStatus.ERROR)):
cause = {
None: 'deleted',
JobStatus.CANCELED: 'canceled',
JobStatus.ERROR: 'stopped due to error',
}[None if not job else job.get('status')]
logger.info('Large image thumbnails job %s' % cause)
return
except Exception:
logger.exception('Error with large image create thumbnails job')
job = Job().updateJob(
job, log='Error creating large image thumbnails\n',
status=JobStatus.ERROR)
return
msg = 'Finished creating large image thumbnails (%d checked or created)' % (
checkedOrCreated)
if failedImages:
msg += ' (%d failed, last failure on item %s)' % (
failedImages, lastFailed)
logger.info(msg)
job = Job().updateJob(job, log=msg + '\n', status=JobStatus.SUCCESS)
def onJobUpdate(event):
"""
Event handler for job update event.
When a Danesfield job succeeds, advance the workflow.
When a Danesfield job fails, remove its associated information
from the workflow manager.
"""
job = event.info['job']
params = event.info['params']
try:
jobId = job[DanesfieldJobKey.ID]
stepName = job[DanesfieldJobKey.STEP_NAME]
except KeyError:
return
try:
# FIXME: Sometimes status is unicode, not int
status = int(params['status'])
except (TypeError, KeyError, ValueError):
return
workflowManager = DanesfieldWorkflowManager.instance()
with _onJobUpdateLock:
# Handle composite steps
if workflowManager.isCompositeStep(jobId, stepName):
# Notify workflow manager when a job in a composite step completes
if status in (JobStatus.SUCCESS, JobStatus.ERROR,
JobStatus.CANCELED):
workflowManager.compositeStepJobCompleted(jobId, stepName)
# Skip processing until all jobs in a composite step have completed
if not workflowManager.isCompositeStepComplete(jobId, stepName):
return
# Set overall status for composite step
successful = workflowManager.isCompositeStepSuccessful(
jobId, stepName)
status = JobStatus.SUCCESS if successful else JobStatus.ERROR
if status == JobStatus.SUCCESS:
# Add standard output from job
# TODO: Alternatively, could record job model ID and defer
# log lookup
# TODO: This currently doesn't support composite steps;
# only the output
# from the last job is saved
job = Job().load(job['_id'], includeLog=True, force=True)
workflowManager.addStandardOutput(jobId=jobId,
stepName=stepName,
output=job.get('log'))
workflowManager.stepSucceeded(jobId=jobId, stepName=stepName)
# Advance workflow asynchronously to avoid affecting
# finished job in case of error
events.daemon.trigger(info={
'jobId': jobId,
'stepName': stepName,
'userId': job['userId']
},
callback=advanceWorkflow)
elif status in (JobStatus.CANCELED, JobStatus.ERROR):
workflowManager.stepFailed(jobId=jobId, stepName=stepName)
def createThumbnailsJob(job):
"""
Create thumbnails for all of the large image items.
:param job: the job object including kwargs which contains:
spec: an array, each entry of which is the parameter dictionary for
the model getThumbnail function.
logInterval: the time in seconds between log messages. This also
controls the granularity of cancelling the job.
concurrent: the number of threads to use. 0 for the number of cpus.
"""
job = Job().updateJob(
job, log='Started creating large image thumbnails\n',
status=JobStatus.RUNNING)
concurrency = int(job['kwargs'].get('concurrent', 0))
concurrency = psutil.cpu_count(logical=True) if concurrency < 1 else concurrency
status = {
'checked': 0,
'created': 0,
'failed': 0,
}
spec = job['kwargs']['spec']
logInterval = float(job['kwargs'].get('logInterval', 10))
job = Job().updateJob(job, log='Creating thumbnails (%d concurrent)\n' % concurrency)
nextLogTime = time.time() + logInterval
tasks = []
# This could be switched from ThreadPoolExecutor to ProcessPoolExecutor
# without any other changes. Doing so would probably improve parallel
# performance, but may not work reliably under Python 2.x.
pool = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
try:
# Get a cursor with the list of images
items = Item().find({'largeImage.fileId': {'$exists': True}})
if hasattr(items, 'count'):
status['items'] = items.count()
status['specs'] = len(spec)
nextitem = cursorNextOrNone(items)
while len(tasks) or nextitem is not None:
# Create more tasks than we strictly need so if one finishes before
# we check another will be ready. This is balanced with not
# creating too many to avoid excessive memory use. As such, we
# can't do a simple iteration over the database cursor, as it will
# be exhausted before we are done.
while len(tasks) < concurrency * 4 and nextitem is not None:
tasks.append(pool.submit(createThumbnailsJobTask, nextitem, spec))
nextitem = cursorNextOrNone(items)
# Wait a short time or until the oldest task is complete
try:
tasks[0].result(0.1)
except concurrent.futures.TimeoutError:
pass
# Remove completed tasks from our list, adding their results to the
# status.
for pos in range(len(tasks) - 1, -1, -1):
if tasks[pos].done():
r = tasks[pos].result()
status['created'] += r['created']
status['checked'] += r['checked']
status['failed'] += r['failed']
status['lastFailed'] = r.get('lastFailed', status.get('lastFailed'))
tasks[pos:pos + 1] = []
# Periodically, log the state of the job and check if it was
# deleted or canceled.
if time.time() > nextLogTime:
job, msg = createThumbnailsJobLog(job, status)
# Check if the job was deleted or canceled; if so, quit
job = Job().load(id=job['_id'], force=True)
if not job or job['status'] in (JobStatus.CANCELED, JobStatus.ERROR):
cause = {
None: 'deleted',
JobStatus.CANCELED: 'canceled',
JobStatus.ERROR: 'stopped due to error',
}[None if not job else job.get('status')]
msg = 'Large image thumbnails job %s' % cause
logger.info(msg)
# Cancel any outstanding tasks. If they haven't started,
# they are discarded. Those that have started will still
# run, though.
for task in tasks:
task.cancel()
return
nextLogTime = time.time() + logInterval
except Exception:
logger.exception('Error with large image create thumbnails job')
Job().updateJob(
job, log='Error creating large image thumbnails\n',
status=JobStatus.ERROR)
return
finally:
# Clean up the task pool asynchronously
pool.shutdown(False)
job, msg = createThumbnailsJobLog(job, status, 'Finished: ', JobStatus.SUCCESS)
logger.info(msg)
