def registerMetadata(self, dataset, metadataFile, user, sendMail=False):
"""Register a .csv file containing metadata about images."""
# Check if image metadata is already registered
if self.findOne({'meta.metadataFiles.fileId': metadataFile['_id']}):
raise ValidationException(
'Metadata file is already registered on a dataset.')
# Add image metadata file information to list
now = datetime.datetime.utcnow()
metadataFiles = dataset['meta']['metadataFiles']
metadataFiles.append({
'fileId': metadataFile['_id'],
'userId': user['_id'],
'time': now
})
dataset = self.setMetadata(dataset, {
'metadataFiles': metadataFiles
})
# Send email notification
if sendMail:
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename='registerMetadataNotification.mako',
templateParams={
'host': host,
'dataset': dataset,
'user': user,
'metadataFile': metadataFile,
'date': now.replace(microsecond=0)
},
subject='ISIC Archive: Dataset Metadata Notification')
return dataset
def requestToParticipate(self, study, params):
currentUser = self.getCurrentUser()
# Check if user already requested to participate in the study
if Study().hasParticipationRequest(study, currentUser):
raise ValidationException('User "%s" already requested to participate in the study.' %
currentUser['_id'])
# Check if user is already an annotator in the study
if Study().hasAnnotator(study, currentUser):
raise ValidationException('User "%s" is already part of the study.' %
currentUser['_id'])
Study().addParticipationRequest(study, currentUser)
# Send email notification to study administrators
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Study Administrators',
templateFilename='participateInStudyRequest.mako',
templateParams={
'host': host,
'study': study,
'user': currentUser
},
subject='ISIC Archive: Study Participation Request')
def requestToParticipate(self, study, params):
currentUser = self.getCurrentUser()
# Check if user already requested to participate in the study
if Study().hasParticipationRequest(study, currentUser):
raise ValidationException(
f'User "{currentUser["_id"]}" already requested to participate in the study.'
)
# Check if user is already an annotator in the study
if Study().hasAnnotator(study, currentUser):
raise ValidationException(
f'User "{currentUser["_id"]}" is already part of the study.')
Study().addParticipationRequest(study, currentUser)
# Send email notification to study administrators
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Study Administrators',
templateFilename='participateInStudyRequest.mako',
templateParams={
'host': host,
'study': study,
'user': currentUser
},
subject='ISIC Archive: Study Participation Request')
def registerMetadata(self, dataset, metadataFile, user, sendMail=False):
"""Register a .csv file containing metadata about images."""
# Check if image metadata is already registered
if self.findOne({'meta.metadataFiles.fileId': metadataFile['_id']}):
raise ValidationException(
'Metadata file is already registered on a dataset.')
# Add image metadata file information to list
now = datetime.datetime.utcnow()
dataset['meta']['metadataFiles'].append({
'fileId': metadataFile['_id'],
'userId': user['_id'],
'time': now
})
dataset = Dataset().save(dataset)
# Send email notification
if sendMail:
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename='registerMetadataNotification.mako',
templateParams={
'host': host,
'dataset': dataset,
'user': user,
'metadataFile': metadataFile,
'date': now.replace(microsecond=0)
},
subject='ISIC Archive: Dataset Metadata Notification')
return dataset
def _sendApprovedEmail(self, user):
text = mail_utils.renderTemplate('accountApproved.mako', {
'user': user,
'url': mail_utils.getEmailUrlPrefix()
})
mail_utils.sendMail('Girder: Account approved', text,
[user.get('email')])
def sendIngestionNotification(batchId, failedImages, skippedFilenames, numImages):
batch = Batch().load(batchId)
dataset = Dataset().load(batch['datasetId'], force=True)
user = User().load(batch['creatorId'], force=True)
host = mail_utils.getEmailUrlPrefix()
# TODO: The email should gracefully handle the situation where failedImages or skippedFilenames
# has an excessive amount of items.
params = {
'isOriginalUploader': True,
'host': host,
'dataset': dataset,
# We intentionally leak full user details here, even though all
# email recipients may not have access permissions to the user
'user': user,
'batch': batch,
'failedImages': failedImages,
'skippedFilenames': skippedFilenames,
'numImages': numImages,
}
subject = f'ISIC Archive: Dataset Upload Confirmation - {dataset["name"]}'
templateFilename = 'ingestDatasetConfirmation.mako'
# Mail user
html = mail_utils.renderTemplate(templateFilename, params)
mail_utils.sendMailSync(subject, html, user['email'])
# Mail 'Dataset QC Reviewers' group
params['isOriginalUploader'] = False
sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename=templateFilename,
templateParams=params,
subject=subject,
asynchronous=False)
def postScore(self, submission, params):
# Ensure admin access on the containing challenge phase
phase = self.model('phase',
'challenge').load(submission['phaseId'],
user=self.getCurrentUser(),
exc=True,
level=AccessType.ADMIN)
submission['score'] = json.loads(cherrypy.request.body.read())
submission = self.model('submission', 'covalic').save(submission)
# Delete the scirubg user's job token since the job is now complete.
token = self.getCurrentToken()
self.model('token').remove(token)
user = self.model('user').load(submission['creatorId'], force=True)
challenge = self.model('challenge',
'challenge').load(phase['challengeId'],
force=True)
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
html = mail_utils.renderTemplate(
'covalic.submissionComplete.mako', {
'phase': phase,
'challenge': challenge,
'submission': submission,
'host': covalicHost
})
mail_utils.sendEmail(to=user['email'],
subject='Your submission has been scored',
text=html)
return submission
def postScore(self, submission, params):
# Ensure admin access on the containing challenge phase
phase = self.model('phase', 'challenge').load(
submission['phaseId'], user=self.getCurrentUser(), exc=True,
level=AccessType.ADMIN)
submission['score'] = json.loads(cherrypy.request.body.read())
submission = self.model('submission', 'covalic').save(submission)
# Delete the scirubg user's job token since the job is now complete.
token = self.getCurrentToken()
self.model('token').remove(token)
user = self.model('user').load(submission['creatorId'], force=True)
challenge = self.model('challenge', 'challenge').load(
phase['challengeId'], force=True)
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
html = mail_utils.renderTemplate('covalic.submissionComplete.mako', {
'phase': phase,
'challenge': challenge,
'submission': submission,
'host': covalicHost
})
mail_utils.sendEmail(
to=user['email'], subject='Your submission has been scored',
text=html)
return submission
def sendIngestionNotification(batchId, failedImages, skippedFilenames):
batch = Batch().load(batchId)
dataset = Dataset().load(batch['datasetId'], force=True)
user = User().load(batch['creatorId'], force=True)
host = mail_utils.getEmailUrlPrefix()
# TODO: The email should gracefully handle the situation where failedImages or skippedFilenames
# has an excessive amount of items.
params = {
'isOriginalUploader': True,
'host': host,
'dataset': dataset,
# We intentionally leak full user details here, even though all
# email recipients may not have access permissions to the user
'user': user,
'batch': batch,
'failedImages': failedImages,
'skippedFilenames': skippedFilenames
}
subject = 'ISIC Archive: Dataset Upload Confirmation'
templateFilename = 'ingestDatasetConfirmation.mako'
# Mail user
html = mail_utils.renderTemplate(templateFilename, params)
sendEmail(to=user['email'], subject=subject, text=html)
# Mail 'Dataset QC Reviewers' group
params['isOriginalUploader'] = False
sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename=templateFilename,
templateParams=params,
subject=subject)
def _sendApprovalEmail(self, user):
url = '%s#user/%s' % (mail_utils.getEmailUrlPrefix(), str(user['_id']))
text = mail_utils.renderTemplate('accountApproval.mako', {
'user': user,
'url': url
})
mail_utils.sendMailToAdmins('Girder: Account pending approval', text)
def _sendApprovedEmail(self, user):
text = mail_utils.renderTemplate('accountApproved.mako', {
'user': user,
'url': mail_utils.getEmailUrlPrefix()
})
mail_utils.sendEmail(
to=user.get('email'),
subject='Girder: Account approved',
text=text)
def _sendVerificationEmail(self, user):
token = self.model('token').createToken(
user, days=1, scope=TokenScope.EMAIL_VERIFICATION)
url = '%s#useraccount/%s/verification/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(
token['_id']))
text = mail_utils.renderTemplate('emailVerification.mako',
{'url': url})
mail_utils.sendEmail(to=user.get('email'),
subject='Girder: Email verification',
text=text)
def _sendApprovalEmail(self, user):
url = '%s/#user/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']))
text = mail_utils.renderTemplate('accountApproval.mako', {
'user': user,
'url': url
})
mail_utils.sendEmail(
toAdmins=True,
subject='Girder: Account pending approval',
text=text)
def _sendVerificationEmail(self, user):
token = self.model('token').createToken(
user, days=1, scope=TokenScope.EMAIL_VERIFICATION)
url = '%s/#useraccount/%s/verification/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(token['_id']))
text = mail_utils.renderTemplate('emailVerification.mako', {
'url': url
})
mail_utils.sendEmail(
to=user.get('email'),
subject='Girder: Email verification',
text=text)
def postScore(self, submission, score, params):
# Ensure admin access on the containing challenge phase
phase = self.model('phase', 'covalic').load(
submission['phaseId'], user=self.getCurrentUser(), exc=True,
level=AccessType.ADMIN)
# Record whether submission is being re-scored
rescoring = 'overallScore' in submission
# Save document to trigger computing overall score
submission.pop('overallScore', None)
submission['score'] = score
submission = self.model('submission', 'covalic').save(submission)
# Delete the scoring user's job token since the job is now complete.
token = self.getCurrentToken()
self.model('token').remove(token)
user = self.model('user').load(submission['creatorId'], force=True)
challenge = self.model('challenge', 'covalic').load(
phase['challengeId'], force=True)
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
# Mail user
if not rescoring:
html = mail_utils.renderTemplate(
'covalic.submissionCompleteUser.mako',
{
'phase': phase,
'challenge': challenge,
'submission': submission,
'host': covalicHost
})
mail_utils.sendEmail(
to=user['email'], subject='Your submission has been scored',
text=html)
# Mail admins
emails = sorted(getPhaseUserEmails(
phase, AccessType.WRITE, includeChallengeUsers=True))
html = mail_utils.renderTemplate(
'covalic.submissionCompleteAdmin.mako',
{
'user': user,
'phase': phase,
'challenge': challenge,
'submission': submission,
'host': covalicHost
})
mail_utils.sendEmail(
to=emails, subject='A submission has been scored', text=html)
return self._filterScore(phase, submission, user)
def _sendVerificationEmail(self, user):
from .token import Token
token = Token().createToken(user,
days=1,
scope=TokenScope.EMAIL_VERIFICATION)
url = '%s#useraccount/%s/verification/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(
token['_id']))
text = mail_utils.renderTemplate('emailVerification.mako',
{'url': url})
mail_utils.sendMail('Girder: Email verification', text,
[user.get('email')])
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
administrators accordingly.
"""
if (event.info['job']['type'] == 'covalic_score' and
'status' in event.info['params'] and
int(event.info['params']['status']) == JobStatus.ERROR):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
html = mail_utils.renderTemplate('covalic.submissionError.mako', {
'submissionId': event.info['job']['covalicSubmissionId'],
'host': covalicHost
})
mail_utils.sendEmail(
toAdmins=True, subject='Submission processing error', text=html)
def createAuthorizedUpload(self, folder, params):
try:
if params.get('duration'):
days = int(params.get('duration'))
else:
days = self.model('setting').get(SettingKey.COOKIE_LIFETIME)
except ValueError:
raise ValidationException('Token duration must be an integer, or leave it empty.')
token = self.model('token').createToken(days=days, user=self.getCurrentUser(), scope=(
TOKEN_SCOPE_AUTHORIZED_UPLOAD, 'authorized_upload_folder_%s' % folder['_id']))
url = '%s#authorized_upload/%s/%s' % (
mail_utils.getEmailUrlPrefix(), folder['_id'], token['_id'])
return {'url': url}
def createAuthorizedUpload(self, folder, params):
try:
if params.get('duration'):
days = int(params.get('duration'))
else:
days = Setting().get(SettingKey.COOKIE_LIFETIME)
except ValueError:
raise ValidationException('Token duration must be an integer, or leave it empty.')
token = Token().createToken(days=days, user=self.getCurrentUser(), scope=(
TOKEN_SCOPE_AUTHORIZED_UPLOAD, 'authorized_upload_folder_%s' % folder['_id']))
url = '%s#authorized_upload/%s/%s' % (
mail_utils.getEmailUrlPrefix(), folder['_id'], token['_id'])
return {'url': url}
def inviteUser(self, params):
Token = self.model('token')
User = self.model('user', 'isic_archive')
params = self._decodeParams(params)
self.requireParams(['login', 'email', 'firstName', 'lastName'], params)
if 'validityPeriod' in params:
try:
validityPeriod = float(params['validityPeriod'])
except ValueError:
raise ValidationException('Validity period must be a number.', 'validityPeriod')
else:
validityPeriod = 60.0
currentUser = self.getCurrentUser()
User.requireAdminStudy(currentUser)
newUser = User.createUser(
login=params['login'],
password=None,
email=params['email'],
firstName=params['firstName'],
lastName=params['lastName']
)
token = Token.createToken(
newUser, days=validityPeriod,
scope=[TokenScope.TEMPORARY_USER_AUTH])
inviteUrl = '%s/#user/%s/rsvp/%s' % (
mail_utils.getEmailUrlPrefix(), newUser['_id'], token['_id'])
html = mail_utils.renderTemplate(
'inviteUser.mako',
{
'newUser': newUser,
'inviteUrl': inviteUrl,
})
mail_utils.sendEmail(
to=newUser['email'],
subject='ISIC Archive: Invitation',
text=html)
return {
'newUser': User.filterSummary(newUser, currentUser),
'inviteUrl': inviteUrl
}
def requestCreateDatasetPermission(self, params):
User = self.model('user', 'isic_archive')
Group = self.model('group')
currentUser = self.getCurrentUser()
resp = {}
if User.canCreateDataset(currentUser):
resp['message'] = 'Dataset Contributor access granted.',
resp['extra'] = 'hasPermission'
else:
# Request that user join group
groupName = 'Dataset Contributors'
group = Group.findOne({'name': groupName})
if not group:
raise RestException('Could not load group: %s' % groupName)
resp['message'] = 'Dataset Contributor access requested. An administrator may contact ' \
'you via email (at %s) to process your request.' % currentUser['email']
for request in Group.getFullRequestList(group):
if request['id'] == currentUser['_id']:
# Request for this user is already pending
break
else:
# No request for this user yet
Group.joinGroup(group, currentUser)
# Send email to group moderators and administrators
groupAcl = Group.getFullAccessList(group)
groupModeratorEmails = [
getUserEmail(user)
for user in groupAcl['users']
if user['level'] >= AccessType.WRITE
]
host = mail_utils.getEmailUrlPrefix()
html = mail_utils.renderTemplate(
'datasetContributorRequest.mako',
{
'user': currentUser,
'group': group,
'host': host,
})
mail_utils.sendEmail(
to=groupModeratorEmails,
subject='ISIC Archive: Dataset Contributor Request',
text=html)
return resp
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
administrators accordingly.
"""
if (event.info['job']['type'] == 'covalic_score'
and 'status' in event.info['params']
and int(event.info['params']['status']) == JobStatus.ERROR):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
html = mail_utils.renderTemplate(
'covalic.submissionError.mako', {
'submissionId': event.info['job']['covalicSubmissionId'],
'host': covalicHost
})
mail_utils.sendEmail(toAdmins=True,
subject='Submission processing error',
text=html)
def generateTemporaryPassword(self, email, params):
user = self.model('user').findOne({'email': email.lower()})
if not user:
raise RestException('That email is not registered.')
token = self.model('token').createToken(
user, days=1, scope=TokenScope.TEMPORARY_USER_AUTH)
url = '%s#useraccount/%s/token/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(token['_id']))
html = mail_utils.renderTemplate('temporaryAccess.mako', {
'url': url,
'token': str(token['_id'])
})
mail_utils.sendEmail(to=email, subject='Girder: Temporary access', text=html)
return {'message': 'Sent temporary access email.'}
def generateTemporaryPassword(self, email):
user = self.model('user').findOne({'email': email.lower()})
if not user:
raise RestException('That email is not registered.')
token = self.model('token').createToken(
user, days=1, scope=TokenScope.TEMPORARY_USER_AUTH)
url = '%s#useraccount/%s/token/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(token['_id']))
html = mail_utils.renderTemplate('temporaryAccess.mako', {
'url': url,
'token': str(token['_id'])
})
mail_utils.sendEmail(to=email, subject='Girder: Temporary access', text=html)
return {'message': 'Sent temporary access email.'}
def _ingestZip(self, dataset, zipFile, batch, user, sendMail):
"""
Ingest images from a ZIP file into a dataset.
The images are extracted to a "Pre-review" folder within the dataset folder.
"""
prereviewFolder = Folder().createFolder(
parent=self.imagesFolder(dataset),
name='Pre-review',
parentType='folder',
creator=user,
public=False,
reuseExisting=True)
# Process zip file
# TODO: gracefully clean up after exceptions in handleZip
self._handleZip(dataset, batch, prereviewFolder, user, zipFile)
# Send email confirmations
if sendMail:
host = mail_utils.getEmailUrlPrefix()
params = {
'group': False,
'host': host,
'dataset': dataset,
# We intentionally leak full user details here, even though all
# email recipients may not have access permissions to the user
'user': user,
'batch': batch,
}
subject = 'ISIC Archive: Dataset Upload Confirmation'
templateFilename = 'ingestDatasetConfirmation.mako'
# Mail user
html = mail_utils.renderTemplate(templateFilename, params)
mail_utils.sendEmail(to=user['email'], subject=subject, text=html)
# Mail 'Dataset QC Reviewers' group
params['group'] = True
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename=templateFilename,
templateParams=params,
subject=subject)
def requestCreateDatasetPermission(self, params):
currentUser = self.getCurrentUser()
resp = {}
if User().canCreateDataset(currentUser):
resp['message'] = 'Dataset Contributor access granted.',
resp['extra'] = 'hasPermission'
else:
# Request that user join group
groupName = 'Dataset Contributors'
group = Group().findOne({'name': groupName})
if not group:
raise RestException('Could not load group: %s' % groupName)
resp['message'] = 'Dataset Contributor access requested. An administrator may contact ' \
'you via email (at %s) to process your request.' % currentUser['email']
for request in Group().getFullRequestList(group):
if request['id'] == currentUser['_id']:
# Request for this user is already pending
break
else:
# No request for this user yet
Group().joinGroup(group, currentUser)
# Send email to group moderators and administrators
groupAcl = Group().getFullAccessList(group)
groupModeratorEmails = [
getUserEmail(user) for user in groupAcl['users']
if user['level'] >= AccessType.WRITE
]
if groupModeratorEmails:
host = mail_utils.getEmailUrlPrefix()
html = mail_utils.renderTemplate(
'datasetContributorRequest.mako', {
'user': currentUser,
'group': group,
'host': host,
})
mail_utils.sendEmail(
to=groupModeratorEmails,
subject='ISIC Archive: Dataset Contributor Request',
text=html)
return resp
def registerMetadata(self, dataset, metadataDataStream, filename, user, sendMail=False):
"""Register CSV data containing metadata about images."""
# Store metadata data in a .csv file attached to the dataset
metadataFile = Upload().uploadFromFile(
obj=metadataDataStream,
size=len(metadataDataStream),
name=filename,
parentType='dataset',
parent=dataset,
attachParent=True,
user=user,
mimeType='text/csv'
)
# TODO: remove this once a bug in upstream Girder is fixed
metadataFile['attachedToType'] = ['dataset', 'isic_archive']
metadataFile = File().save(metadataFile)
# Add image metadata file information to list
now = datetime.datetime.utcnow()
dataset['metadataFiles'].append({
'fileId': metadataFile['_id'],
'userId': user['_id'],
'time': now
})
dataset = Dataset().save(dataset)
# Send email notification
if sendMail:
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename='registerMetadataNotification.mako',
templateParams={
'host': host,
'dataset': dataset,
'user': user,
'metadataFile': metadataFile,
'date': now.replace(microsecond=0)
},
subject='ISIC Archive: Dataset Metadata Notification')
return dataset
def registerMetadata(self, dataset, metadataDataStream, filename, user, sendMail=False):
"""Register CSV data containing metadata about images."""
# Store metadata data in a .csv file attached to the dataset
metadataFile = Upload().uploadFromFile(
obj=metadataDataStream,
size=len(metadataDataStream),
name=filename,
parentType='dataset',
parent=dataset,
attachParent=True,
user=user,
mimeType='text/csv'
)
# TODO: remove this once a bug in upstream Girder is fixed
metadataFile['attachedToType'] = ['dataset', 'isic_archive']
metadataFile = File().save(metadataFile)
# Add image metadata file information to list
now = datetime.datetime.utcnow()
dataset['metadataFiles'].append({
'fileId': metadataFile['_id'],
'userId': user['_id'],
'time': now
})
dataset = Dataset().save(dataset)
# Send email notification
if sendMail:
host = mail_utils.getEmailUrlPrefix()
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename='registerMetadataNotification.mako',
templateParams={
'host': host,
'dataset': dataset,
'user': user,
'metadataFile': metadataFile,
'date': now.replace(microsecond=0)
},
subject=f'ISIC Archive: Dataset Metadata Notification - {dataset["name"]}')
return dataset
def generateTemporaryPassword(self, email):
user = self._model.findOne({'email': email.lower()})
if not user:
raise RestException('That email is not registered.')
token = Token().createToken(user, days=1, scope=TokenScope.TEMPORARY_USER_AUTH)
url = '%s#useraccount/%s/token/%s' % (
mail_utils.getEmailUrlPrefix(), str(user['_id']), str(token['_id']))
html = mail_utils.renderTemplate('temporaryAccess.mako', {
'url': url,
'token': str(token['_id'])
})
mail_utils.sendMail(
'%s: Temporary access' % Setting().get(SettingKey.BRAND_NAME),
html,
[email]
)
return {'message': 'Sent temporary access email.'}
def inviteUser(self, params):
params = self._decodeParams(params)
self.requireParams(['login', 'email', 'firstName', 'lastName'], params)
if 'validityPeriod' in params:
try:
validityPeriod = float(params['validityPeriod'])
except ValueError:
raise ValidationException('Validity period must be a number.',
'validityPeriod')
else:
validityPeriod = 60.0
currentUser = self.getCurrentUser()
User().requireAdminStudy(currentUser)
newUser = User().createUser(login=params['login'],
password=None,
email=params['email'],
firstName=params['firstName'],
lastName=params['lastName'])
token = Token().createToken(newUser,
days=validityPeriod,
scope=[TokenScope.TEMPORARY_USER_AUTH])
inviteUrl = '%s/#user/%s/rsvp/%s' % (mail_utils.getEmailUrlPrefix(),
newUser['_id'], token['_id'])
html = mail_utils.renderTemplate('inviteUser.mako', {
'newUser': newUser,
'inviteUrl': inviteUrl,
})
mail_utils.sendEmail(to=newUser['email'],
subject='ISIC Archive: Invitation',
text=html)
return {
'newUser': User().filterSummary(newUser, currentUser),
'inviteUrl': inviteUrl
}
def deleteExpired(self):
cursor = Folder().find({'isPhotomorph': True})
now = datetime.datetime.utcnow()
emailExp = datetime.timedelta(days=DAYS_UNTIL_EMAIL)
dataExp = datetime.timedelta(days=DAYS_UNTIL_DELETION)
for folder in cursor:
setResponseTimeLimit()
if folder['created'] + dataExp < now:
logger.info('Delete timelapse %s (uid=%s)' %
(folder['name'], folder['creatorId']))
Folder().remove(folder)
elif not folder.get('timelapseEmailSent'
) and folder['created'] + emailExp < now:
try:
user = User().load(folder['creatorId'],
force=True,
exc=True)
text = renderTemplate(
'timelapse.deletePending.mako',
params={
'folder':
folder,
'days':
DAYS_UNTIL_DELETION,
'url':
getEmailUrlPrefix() + '#timelapse',
'deletionDate':
(folder['created'] + dataExp).strftime(DATE_FMT)
})
sendMail(DELETE_SUBJECT, text, [user['email']])
Folder().update({'_id': folder['_id']},
{'$set': {
'timelapseEmailSent': True
}},
multi=False)
except Exception:
logger.exception('Error sending email for folder: %s' %
folder['_id'])
def ingestDataset(self,
zipFile,
user,
name,
owner,
description,
license,
signature,
anonymous,
attribution,
sendMail=False):
"""
Ingest an uploaded dataset from a .zip file of images. The images are
extracted to a "Pre-review" folder within a new dataset folder.
"""
# Create dataset folder
dataset = self.createDataset(name, description, user)
# Set dataset metadata, including license info
dataset['meta'] = {
'owner': owner,
'signature': signature,
'anonymous': anonymous,
'attribution': attribution,
'license': license,
'metadataFiles': []
}
dataset = Dataset().save(dataset)
prereviewFolder = Folder().createFolder(parent=dataset,
name='Pre-review',
parentType='folder',
creator=user,
public=False)
prereviewFolder = Folder().copyAccessPolicies(dataset,
prereviewFolder,
save=True)
# Process zip file
# TODO: gracefully clean up after exceptions in handleZip
self._handleZip(prereviewFolder, user, zipFile)
# Send email confirmations
if sendMail:
host = mail_utils.getEmailUrlPrefix()
params = {
'group': False,
'host': host,
# We intentionally leak full user details here, even though all
# email recipients may not have access permissions to the user
'user': user,
'name': name,
'owner': owner,
'description': description,
'license': license,
'signature': signature,
'attribution': 'Anonymous' if anonymous else attribution
}
subject = 'ISIC Archive: Dataset Upload Confirmation'
templateFilename = 'ingestDatasetConfirmation.mako'
# Mail user
html = mail_utils.renderTemplate(templateFilename, params)
mail_utils.sendEmail(to=user['email'], subject=subject, text=html)
# Mail 'Dataset QC Reviewers' group
params['group'] = True
isic_mail_utils.sendEmailToGroup(groupName='Dataset QC Reviewers',
templateFilename=templateFilename,
templateParams=params,
subject=subject)
return dataset
def ingestDataset(self, zipFile, user, name, owner, description,
license, signature, anonymous, attribution,
sendMail=False):
"""
Ingest an uploaded dataset from a .zip file of images. The images are
extracted to a "Pre-review" folder within a new dataset folder.
"""
Folder = self.model('folder')
# Create dataset folder
dataset = self.createDataset(name, description, user)
# Set dataset metadata, including license info
dataset = self.setMetadata(dataset, {
'owner': owner,
'signature': signature,
'anonymous': anonymous,
'attribution': attribution,
'license': license,
'metadataFiles': []
})
prereviewFolder = Folder.createFolder(
parent=dataset,
name='Pre-review',
parentType='folder',
creator=user,
public=False)
prereviewFolder = Folder.copyAccessPolicies(
dataset, prereviewFolder, save=True)
# Process zip file
# TODO: gracefully clean up after exceptions in handleZip
self._handleZip(prereviewFolder, user, zipFile)
# Send email confirmations
if sendMail:
host = mail_utils.getEmailUrlPrefix()
params = {
'group': False,
'host': host,
# We intentionally leak full user details here, even though all
# email recipients may not have access permissions to the user
'user': user,
'name': name,
'owner': owner,
'description': description,
'license': license,
'signature': signature,
'attribution': 'Anonymous' if anonymous else attribution
}
subject = 'ISIC Archive: Dataset Upload Confirmation'
templateFilename = 'ingestDatasetConfirmation.mako'
# Mail user
html = mail_utils.renderTemplate(templateFilename, params)
mail_utils.sendEmail(to=user['email'], subject=subject, text=html)
# Mail 'Dataset QC Reviewers' group
params['group'] = True
isic_mail_utils.sendEmailToGroup(
groupName='Dataset QC Reviewers',
templateFilename=templateFilename,
templateParams=params,
subject=subject)
return dataset
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
the user and challenge/phase administrators accordingly. Here, an
administrator is defined to be a user with WRITE access or above.
"""
isErrorStatus = False
try:
isErrorStatus = int(
event.info['params'].get('status')) == JobStatus.ERROR
except (ValueError, TypeError):
pass
if (event.info['job']['type'] == 'covalic_score' and isErrorStatus):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
# Create minimal log that contains only Covalic errors.
# Use full log if no Covalic-specific errors are found.
# Fetch log from model, because log in event may not be up-to-date.
job = ModelImporter.model('job', 'jobs').load(event.info['job']['_id'],
includeLog=True,
force=True)
log = job.get('log')
minimalLog = None
if log:
log = ''.join(log)
minimalLog = '\n'.join([
line[len(JOB_LOG_PREFIX):].strip()
for line in log.splitlines() if line.startswith(JOB_LOG_PREFIX)
])
if not minimalLog:
minimalLog = log
submission = ModelImporter.model('submission', 'covalic').load(
event.info['job']['covalicSubmissionId'])
phase = ModelImporter.model('phase',
'covalic').load(submission['phaseId'],
force=True)
challenge = ModelImporter.model('challenge',
'covalic').load(phase['challengeId'],
force=True)
user = ModelImporter.model('user').load(event.info['job']['userId'],
force=True)
rescoring = job.get('rescoring', False)
# Mail admins, include full log
emails = sorted(
getPhaseUserEmails(phase,
AccessType.WRITE,
includeChallengeUsers=True))
html = mail_utils.renderTemplate(
'covalic.submissionErrorAdmin.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'user': user,
'host': covalicHost,
'log': log
})
mail_utils.sendEmail(to=emails,
subject='Submission processing error',
text=html)
# Mail user, include minimal log
if not rescoring:
html = mail_utils.renderTemplate(
'covalic.submissionErrorUser.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'host': covalicHost,
'log': minimalLog
})
mail_utils.sendEmail(to=user['email'],
subject='Submission processing error',
text=html)
def onJobUpdate(event):
"""
Hook into job update event so we can look for job failure events and email
the user and challenge/phase administrators accordingly. Here, an
administrator is defined to be a user with WRITE access or above.
"""
isErrorStatus = False
try:
isErrorStatus = int(event.info['params'].get('status')) == JobStatus.ERROR
except (ValueError, TypeError):
pass
if (event.info['job']['type'] == 'covalic_score' and isErrorStatus):
covalicHost = posixpath.dirname(mail_utils.getEmailUrlPrefix())
# Create minimal log that contains only Covalic errors.
# Use full log if no Covalic-specific errors are found.
# Fetch log from model, because log in event may not be up-to-date.
job = Job().load(
event.info['job']['_id'], includeLog=True, force=True)
log = job.get('log')
minimalLog = None
if log:
log = ''.join(log)
minimalLog = '\n'.join([line[len(JOB_LOG_PREFIX):].strip()
for line in log.splitlines()
if line.startswith(JOB_LOG_PREFIX)])
if not minimalLog:
minimalLog = log
submission = Submission().load(
event.info['job']['covalicSubmissionId'])
phase = Phase().load(
submission['phaseId'], force=True)
challenge = Challenge().load(
phase['challengeId'], force=True)
user = User().load(
event.info['job']['userId'], force=True)
rescoring = job.get('rescoring', False)
# Mail admins, include full log
emails = sorted(getPhaseUserEmails(
phase, AccessType.WRITE, includeChallengeUsers=True))
html = mail_utils.renderTemplate('covalic.submissionErrorAdmin.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'user': user,
'host': covalicHost,
'log': log
})
mail_utils.sendEmail(
to=emails, subject='Submission processing error', text=html)
# Mail user, include minimal log
if not rescoring:
html = mail_utils.renderTemplate('covalic.submissionErrorUser.mako', {
'submission': submission,
'challenge': challenge,
'phase': phase,
'host': covalicHost,
'log': minimalLog
})
mail_utils.sendEmail(
to=user['email'], subject='Submission processing error', text=html)
