def deleteIncompleteTiles(self, params):
result = {'removed': 0}
while True:
item = Item().findOne({'largeImage.expected': True})
if not item:
break
job = Job().load(item['largeImage']['jobId'], force=True)
if job and job.get('status') in (JobStatus.QUEUED,
JobStatus.RUNNING):
job = Job().cancelJob(job)
if job and job.get('status') in (JobStatus.QUEUED,
JobStatus.RUNNING):
result['message'] = ('The job for item %s could not be '
'canceled' % (str(item['_id'])))
break
ImageItem().delete(item)
result['removed'] += 1
return result
def delete(self, item, skipFileIds=None):
deleted = False
if 'largeImage' in item:
job = None
if 'jobId' in item['largeImage']:
try:
job = Job().load(item['largeImage']['jobId'],
force=True,
exc=True)
except ValidationException:
# The job has been deleted, but we still need to clean up
# the rest of the tile information
pass
if (item['largeImage'].get('expected') and job
and job.get('status')
in (JobStatus.QUEUED, JobStatus.RUNNING)):
# cannot cleanly remove the large image, since a conversion
# job is currently in progress
# TODO: cancel the job
# TODO: return a failure error code
return False
# If this file was created by the worker job, delete it
if 'jobId' in item['largeImage']:
# To eliminate all traces of the job, add
# if job:
# Job().remove(job)
del item['largeImage']['jobId']
if 'originalId' in item['largeImage']:
# The large image file should not be the original file
assert item['largeImage']['originalId'] != \
item['largeImage'].get('fileId')
if ('fileId' in item['largeImage'] and
(not skipFileIds
or item['largeImage']['fileId'] not in skipFileIds)):
file = File().load(id=item['largeImage']['fileId'],
force=True)
if file:
File().remove(file)
del item['largeImage']['originalId']
del item['largeImage']
item = self.save(item)
deleted = True
self.removeThumbnailFiles(item)
return deleted
def createThumbnailsJob(job):
"""
Create thumbnails for all of the large image items.
:param job: the job object including kwargs which contains:
spec: an array, each entry of which is the parameter dictionary for
the model getThumbnail function.
logInterval: the time in seconds between log messages. This also
controls the granularity of cancelling the job.
concurrent: the number of threads to use. 0 for the number of cpus.
"""
job = Job().updateJob(job,
log='Started creating large image thumbnails\n',
status=JobStatus.RUNNING)
concurrency = int(job['kwargs'].get('concurrent', 0))
concurrency = psutil.cpu_count(
logical=True) if concurrency < 1 else concurrency
status = {
'checked': 0,
'created': 0,
'failed': 0,
}
spec = job['kwargs']['spec']
logInterval = float(job['kwargs'].get('logInterval', 10))
job = Job().updateJob(job,
log='Creating thumbnails (%d concurrent)\n' %
concurrency)
nextLogTime = time.time() + logInterval
tasks = []
# This could be switched from ThreadPoolExecutor to ProcessPoolExecutor
# without any other changes. Doing so would probably improve parallel
# performance, but may not work reliably under Python 2.x.
pool = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
try:
# Get a cursor with the list of images
items = Item().find({'largeImage.fileId': {'$exists': True}})
if hasattr(items, 'count'):
status['items'] = items.count()
status['specs'] = len(spec)
nextitem = cursorNextOrNone(items)
while len(tasks) or nextitem is not None:
# Create more tasks than we strictly need so if one finishes before
# we check another will be ready. This is balanced with not
# creating too many to avoid excessive memory use. As such, we
# can't do a simple iteration over the database cursor, as it will
# be exhausted before we are done.
while len(tasks) < concurrency * 4 and nextitem is not None:
tasks.append(
pool.submit(createThumbnailsJobTask, nextitem, spec))
nextitem = cursorNextOrNone(items)
# Wait a short time or until the oldest task is complete
try:
tasks[0].result(0.1)
except concurrent.futures.TimeoutError:
pass
# Remove completed tasks from our list, adding their results to the
# status.
for pos in range(len(tasks) - 1, -1, -1):
if tasks[pos].done():
r = tasks[pos].result()
status['created'] += r['created']
status['checked'] += r['checked']
status['failed'] += r['failed']
status['lastFailed'] = r.get('lastFailed',
status.get('lastFailed'))
tasks[pos:pos + 1] = []
# Periodically, log the state of the job and check if it was
# deleted or canceled.
if time.time() > nextLogTime:
job, msg = createThumbnailsJobLog(job, status)
# Check if the job was deleted or canceled; if so, quit
job = Job().load(id=job['_id'], force=True)
if not job or job['status'] in (JobStatus.CANCELED,
JobStatus.ERROR):
cause = {
None: 'deleted',
JobStatus.CANCELED: 'canceled',
JobStatus.ERROR: 'stopped due to error',
}[None if not job else job.get('status')]
msg = 'Large image thumbnails job %s' % cause
logger.info(msg)
# Cancel any outstanding tasks. If they haven't started,
# they are discarded. Those that have started will still
# run, though.
for task in tasks:
task.cancel()
return
nextLogTime = time.time() + logInterval
except Exception:
logger.exception('Error with large image create thumbnails job')
Job().updateJob(job,
log='Error creating large image thumbnails\n',
status=JobStatus.ERROR)
return
finally:
# Clean up the task pool asynchronously
pool.shutdown(False)
job, msg = createThumbnailsJobLog(job, status, 'Finished: ',
JobStatus.SUCCESS)
logger.info(msg)
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
the user and challenge/phase administrators accordingly. Here, an
administrator is defined to be a user with WRITE access or above.
"""
isErrorStatus = False
try:
isErrorStatus = int(event.info['params'].get('status')) == JobStatus.ERROR
except (ValueError, TypeError):
pass
if (event.info['job']['type'] == 'covalic_score' and isErrorStatus):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
# Create minimal log that contains only Covalic errors.
# Use full log if no Covalic-specific errors are found.
# Fetch log from model, because log in event may not be up-to-date.
job = Job().load(
event.info['job']['_id'], includeLog=True, force=True)
log = job.get('log')
minimalLog = None
if log:
log = ''.join(log)
minimalLog = '\n'.join([line[len(JOB_LOG_PREFIX):].strip()
for line in log.splitlines()
if line.startswith(JOB_LOG_PREFIX)])
if not minimalLog:
minimalLog = log
submission = Submission().load(
event.info['job']['covalicSubmissionId'])
phase = Phase().load(
submission['phaseId'], force=True)
challenge = Challenge().load(
phase['challengeId'], force=True)
user = User().load(
event.info['job']['userId'], force=True)
rescoring = job.get('rescoring', False)
# Mail admins, include full log
emails = sorted(getPhaseUserEmails(
phase, AccessType.WRITE, includeChallengeUsers=True))
html = mail_utils.renderTemplate('covalic.submissionErrorAdmin.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'user': user,
'host': covalicHost,
'log': log
})
mail_utils.sendEmail(
to=emails, subject='Submission processing error', text=html)
# Mail user, include minimal log
if not rescoring:
html = mail_utils.renderTemplate('covalic.submissionErrorUser.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'host': covalicHost,
'log': minimalLog
})
mail_utils.sendEmail(
to=user['email'], subject='Submission processing error', text=html)
