def addImages(self, study, params):
params = self._decodeParams(params)
self.requireParams(['imageIds'], params)
creatorUser = self.getCurrentUser()
User().requireAdminStudy(creatorUser)
# Load all images before adding any, to ensure all are valid and
# accessible
if len(set(params['imageIds'])) != len(params['imageIds']):
raise ValidationException('Duplicate image IDs.', 'imageIds')
images = [
# TODO: This should probably not allow images that the user only as access to via an
# annotation
Image().load(imageId, user=creatorUser, level=AccessType.READ, exc=True)
for imageId in params['imageIds']
]
duplicateAnnotations = Annotation().find({
'meta.studyId': study['_id'],
'meta.imageId': {'$in': [image['_id'] for image in images]}
})
if duplicateAnnotations.count():
# Just list the first duplicate
duplicateAnnotation = next(iter(duplicateAnnotations))
raise ValidationException(
'Image "%s" is already part of the study.' % duplicateAnnotation['meta']['imageId'])
for image in images:
Study().addImage(study, image, creatorUser)
return self.getStudy(id=study['_id'], params={})
def _clientUploadChunk(self, upload, chunk):
"""
Clients that support direct-to-S3 upload behavior will go through this
method by sending a normally-encoded form string as the chunk parameter,
containing the required JSON info for uploading. This generates the
signed URL that the client should use to upload the chunk to S3.
"""
info = json.loads(chunk)
index = int(info['partNumber']) - 1
length = min(self.CHUNK_LEN, upload['size'] - index * self.CHUNK_LEN)
if 'contentLength' in info and int(info['contentLength']) != length:
raise ValidationException('Expected chunk size %d, but got %d.' %
(length, info['contentLength']))
if length <= 0:
raise ValidationException('Invalid chunk length %d.' % length)
queryParams = {
'partNumber': info['partNumber'],
'uploadId': info['s3UploadId']
}
url = self._botoGenerateUrl(method='PUT',
key=upload['s3']['key'],
queryParams=queryParams,
headers={'Content-Length': length})
upload['s3']['uploadId'] = info['s3UploadId']
upload['s3']['partNumber'] = info['partNumber']
upload['s3']['request'] = {'method': 'PUT', 'url': url}
return upload
def validate(self, doc):
starttime = time.time()
try:
# This block could just use the json validator:
# jsonschema.validate(doc.get('annotation'),
# AnnotationSchema.annotationSchema)
# but this is very slow. Instead, validate the main structure and
# then validate each element. If sequential elements are similar
# in structure, skip validating them.
annot = doc.get('annotation')
elements = annot.get('elements', [])
annot['elements'] = []
self.validatorAnnotation.validate(annot)
lastValidatedElement = None
for element in elements:
if not self._similarElementStructure(element,
lastValidatedElement):
self.validatorAnnotationElement.validate(element)
lastValidatedElement = element
annot['elements'] = elements
except jsonschema.ValidationError as exp:
raise ValidationException(exp)
logger.debug('Validated in %5.3fs' % (time.time() - starttime))
elementIds = [
entry['id'] for entry in doc['annotation'].get('elements', [])
if 'id' in entry
]
if len(set(elementIds)) != len(elementIds):
raise ValidationException('Annotation Element IDs are not unique')
return doc
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
"""
if 'prefix' not in doc:
doc['prefix'] = ''
while len(doc['prefix']) and doc['prefix'][0] == '/':
doc['prefix'] = doc['prefix'][1:]
while len(doc['prefix']) and doc['prefix'][-1] == '/':
doc['prefix'] = doc['prefix'][:-1]
if not doc.get('bucket'):
raise ValidationException('Bucket must not be empty.', 'bucket')
if not doc.get('secret'):
raise ValidationException('Secret key must not be empty.',
'secretKey')
if not doc.get('accessKeyId'):
raise ValidationException('Access key ID must not be empty.',
'accessKeyId')
# Make sure we can write into the given bucket using boto
try:
conn = boto.connect_s3(aws_access_key_id=doc['accessKeyId'],
aws_secret_access_key=doc['secret'])
bucket = conn.lookup(bucket_name=doc['bucket'], validate=False)
testKey = boto.s3.key.Key(bucket=bucket,
name=os.path.join(doc['prefix'], 'test'))
testKey.set_contents_from_string('')
except:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to write into bucket "{}".'.format(doc['bucket']),
'bucket')
return doc
def importData(self, parent, parentType, params, progress, user):
importPath = params['importPath']
if not os.path.isdir(importPath):
raise ValidationException('No such directory: %s.' % importPath)
for name in os.listdir(importPath):
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
folder = self.model('folder').createFolder(
parent=parent,
name=name,
parentType=parentType,
creator=user,
reuseExisting=True)
self.importData(
folder,
'folder',
params={'importPath': os.path.join(importPath, name)},
progress=progress,
user=user)
else:
if parentType != 'folder':
raise ValidationException(
'Files cannot be imported directly underneath a %s.' %
parentType)
item = self.model('item').createItem(name=name,
creator=user,
folder=parent,
reuseExisting=True)
self.importFile(item, path, user, name=name)
def validate(self, doc):
"""Validate this Jobuser object."""
doc['processedFiles'] = doc.get('processedFiles', [])
for modelType, key in ((('job', 'jobs'), 'jobId'), (('user', ),
'userId')):
childId = doc.get(key)
if not childId:
raise ValidationException(
'Jobuser {} must not be empty.'.format(key), key)
childDoc = self.model(*modelType).findOne({'_id': childId},
fields=['_id'])
if not childDoc:
raise ValidationException(
'Jobuser referenced {} not found: {}.'.format(
modelType[0], childId), key)
for processedFile in doc['processedFiles']:
for key in ('fileId', 'itemId', 'name'):
if key not in processedFile:
raise ValidationException(
'Jobuser processedFile entry must have a {}.'.format(
key), 'processedFile')
return doc
def downloadMultiple(self, params):
include = params.get('include', 'all')
if include not in {'all', 'images', 'metadata'}:
raise ValidationException(
'Param "include" must be one of: "all", "images", "metadata"', 'include')
if 'filter' in params:
query = self._parseFilter(params['filter'])
else:
query = {}
if 'datasetId' in params:
try:
query.update({'folderId': ObjectId(params['datasetId'])})
except InvalidId:
raise ValidationException('Invalid ObjectId.', 'datasetId')
user = self.getCurrentUser()
downloadFileName = 'ISIC-images'
images = Image().filterResultsByPermission(
Image().find(query, sort=[('name', 1)]),
user=user, level=AccessType.READ, limit=0, offset=0)
imagesZipGenerator = self._imagesZipGenerator(downloadFileName, images, include)
setResponseHeader('Content-Type', 'application/zip')
setResponseHeader('Content-Disposition', 'attachment; filename="%s.zip"' % downloadFileName)
return lambda: imagesZipGenerator
def createFeatureset(self, params):
creatorUser = self.getCurrentUser()
# For now, study admins will be the ones that can create featuresets
User().requireAdminStudy(creatorUser)
params = self._decodeParams(params)
self.requireParams(
['name', 'version', 'globalFeatures', 'localFeatures'], params)
featuresetName = params['name'].strip()
if not featuresetName:
raise ValidationException('Name must not be empty.', 'name')
try:
featuresetVersion = float(params['version'])
except ValueError:
raise ValidationException('Version must be a number.', 'version')
# These will be validated once the new featureset is created
globalFeatures = params['globalFeatures']
localFeatures = params['localFeatures']
featureset = Featureset().createFeatureset(
name=featuresetName,
version=featuresetVersion,
creator=creatorUser,
globalFeatures=globalFeatures,
localFeatures=localFeatures)
return self.getFeatureset(id=featureset['_id'], params={})
def validate(self, project):
"""
Validate using jsonschema
"""
try:
ref_resolver = jsonschema.RefResolver.from_schema(
schema.definitions)
jsonschema.validate(project, schema.project, resolver=ref_resolver)
except jsonschema.ValidationError as ve:
raise ValidationException(ve.message)
# Ensure unique name for the project
user = getCurrentUser()
q = {
'name': project['name'],
'userId': user['_id']
}
if '_id' in project:
q['_id'] = {'$ne': project['_id']}
duplicate = self.findOne(q, fields=['_id'])
if duplicate is not None:
raise ValidationException('A project with that name already '
'exists.', 'name')
return project
def validateCullingPeriod(doc):
try:
float(doc['value'])
except KeyError:
raise ValidationException('Culling period must not be empty.', 'value')
except ValueError:
raise ValidationException('Culling period must float.', 'value')
def validateSettings(event):
"""
Handle plugin-specific system settings. Right now we don't do any
validation for the broker or backend URL settings, but we do reinitialize
the celery app object with the new values.
"""
key = event.info['key']
val = event.info['value']
if key == FlowPluginSettings.FULL_ACCESS_USERS:
if not isinstance(val, (list, tuple)):
raise ValidationException('Full access users must be a JSON list.')
event.preventDefault()
elif key == FlowPluginSettings.FULL_ACCESS_GROUPS:
if not isinstance(val, (list, tuple)):
raise ValidationException(
'Full access groups must be a JSON list.')
event.preventDefault()
elif key == FlowPluginSettings.REQUIRE_AUTH:
if not isinstance(val, bool):
raise ValidationException(
'Require auth setting must be true or false.')
event.preventDefault()
elif key == FlowPluginSettings.SAFE_FOLDERS:
if not isinstance(val, (list, tuple)):
raise ValidationException('Safe folders must be a JSON list.')
event.preventDefault()
def _transformOutputs(self, outputs, token, job, task, taskId):
"""
Validates and sanitizes the output bindings. If they are Girder outputs, adds
the necessary token info. If the token does not allow DATA_WRITE, or if the user
does not have write access to the destination, raises an AccessException.
"""
transformed = {}
for k, v in six.viewitems(outputs):
if v['mode'] == 'girder':
ensureTokenScopes(token, TokenScope.DATA_WRITE)
ptype = v.get('parent_type', 'folder')
if not self._validateOutputParentType(k, ptype, task['outputs']):
raise ValidationException('Invalid output parent type: %s.' % ptype)
parent = self.model(ptype).load(
v['parent_id'], level=AccessType.WRITE, user=self.getCurrentUser(), exc=True)
transformed[k] = utils.girderOutputSpec(
parent, parentType=ptype, token=token, name=v.get('name'), dataFormat='none',
reference=json.dumps({
'type': 'item_tasks.output',
'id': k,
'jobId': str(job['_id']),
'taskId': str(taskId)
}))
else:
raise ValidationException('Invalid output mode: %s.' % v['mode'])
return transformed
def validate(self, cluster):
if not cluster['name']:
raise ValidationException('Name must not be empty.', 'name')
if not cluster['type']:
raise ValidationException('Type must not be empty.', 'type')
scheduler_type = parse('config.scheduler.type').find(cluster)
if scheduler_type:
scheduler_type = scheduler_type[0].value
else:
scheduler_type = QueueType.SGE
config = cluster.setdefault('config', {})
scheduler = config.setdefault('scheduler', {})
scheduler['type'] = scheduler_type
if not queue.is_valid_type(scheduler_type):
raise ValidationException('Unsupported scheduler.', 'type')
# If inserting, ensure no other clusters have the same name field amd
# type
if '_id' not in cluster:
query = {
'name': cluster['name'],
'userId': getCurrentUser()['_id'],
'type': cluster['type']
}
if self.findOne(query):
raise ValidationException('A cluster with that name already '
'exists', 'name')
adapter = get_cluster_adapter(cluster)
return adapter.validate()
def _transformInputs(self, inputs, token):
"""
Validates and sanitizes the input bindings. If they are Girder inputs, adds
the necessary token info. If the token does not allow DATA_READ, or if the user
does not have read access to the resource, raises an AccessException.
"""
transformed = {}
for k, v in six.viewitems(inputs):
if v['mode'] == 'girder':
ensureTokenScopes(token, TokenScope.DATA_READ)
rtype = v.get('resource_type', 'file')
if rtype not in {'file', 'item', 'folder'}:
raise ValidationException('Invalid input resource_type: %s.' % rtype)
resource = self.model(rtype).load(
v['id'], level=AccessType.READ, user=self.getCurrentUser(), exc=True)
transformed[k] = utils.girderInputSpec(
resource, resourceType=rtype, token=token, dataFormat='none')
elif v['mode'] == 'inline':
transformed[k] = {
'mode': 'inline',
'data': v['data']
}
else:
raise ValidationException('Invalid input mode: %s.' % v['mode'])
return transformed
def validate(self):
profile_id = parse('profileId').find(self.volume)[0].value
profile = self.model('aws', 'cumulus').load(profile_id,
user=getCurrentUser())
if not profile:
raise ValidationException('Invalid profile id')
valid_fs = ['ext2', 'ext3', 'ext4']
if 'fs' in self.volume and self.volume['fs'] not in valid_fs:
raise ValidationException('Unsupported file system type', 'fs')
try:
int(self.volume['size'])
except ValueError:
raise ValidationException('size number in an integer', 'size')
# Name should be unique
user = getCurrentUser()
query = {
'name': self.volume['name'],
'userId': user['_id']
}
if '_id' in self.volume:
query['_id'] = {'$ne': self.volume['_id']}
volume = self.model('volume', 'cumulus').findOne(query)
if volume:
raise ValidationException('A volume with that name already exists',
'name')
return self.volume
def validate(self, doc):
if not doc.get('name'):
raise ValidationException('Phase name must not be empty.',
field='name')
if not isinstance(doc.get('challengeId'), ObjectId):
raise ValidationException(
'Must have a challenge ID for the phase.', field='challengeId')
if doc.get('startDate'):
doc['startDate'] = validateDate(doc['startDate'], 'startDate')
if doc.get('endDate'):
doc['endDate'] = validateDate(doc['endDate'], 'endDate')
# Check that dates are in a sensible order
if doc.get('startDate') and doc.get('endDate'):
if doc['startDate'] >= doc['endDate']:
raise ValidationException('Invalid start and end dates.',
field='startDate')
# Ensure dockerArgs is a proper JSON list. If not, convert it to one.
if doc.get('scoreTask', {}).get('dockerArgs'):
args = doc['scoreTask']['dockerArgs']
if isinstance(args, six.string_types):
try:
doc['scoreTask']['dockerArgs'] = json.loads(args)
except ValueError:
raise ValidationException(
'Docker arguments must be specified as a JSON list.')
if not isinstance(doc['scoreTask']['dockerArgs'], list):
raise ValidationException('Docker arguments must be a list.')
return doc
def validateInfo(doc):
"""
Validate the assetstore -- make sure we can connect to it and that the
necessary indexes are set up.
"""
if not doc.get('db', ''):
raise ValidationException('Database name must not be empty.', 'db')
if '.' in doc['db'] or ' ' in doc['db']:
raise ValidationException(
'Database name cannot contain spaces'
' or periods.', 'db')
try:
chunkColl = getDbConnection(
doc.get('mongohost', None),
doc.get('replicaset', None),
autoRetry=False,
serverSelectionTimeoutMS=10000)[doc['db']].chunk
chunkColl.create_index([('uuid', pymongo.ASCENDING),
('n', pymongo.ASCENDING)],
unique=True)
except pymongo.errors.ServerSelectionTimeoutError as e:
raise ValidationException('Could not connect to the database: %s' %
str(e))
return doc
def doSegmentation(self, image, params):
params = self._decodeParams(params)
self.requireParams(['seed', 'tolerance'], params)
# validate parameters
seedCoord = params['seed']
if not (
isinstance(seedCoord, list) and
len(seedCoord) == 2 and
all(isinstance(value, int) for value in seedCoord)
):
raise ValidationException('Value must be a coordinate pair.', 'seed')
tolerance = params['tolerance']
if not isinstance(tolerance, int):
raise ValidationException('Value must be an integer.', 'tolerance')
try:
contourCoords = Segmentation().doContourSegmentation(image, seedCoord, tolerance)
except GirderException as e:
raise RestException(e.message)
contourFeature = geojson.Feature(
geometry=geojson.Polygon(
coordinates=(contourCoords.tolist(),)
),
properties={
'source': 'autofill',
'seedPoint': seedCoord,
'tolerance': tolerance
}
)
return contourFeature
def addAnnotators(self, study, params):
# TODO: make the loadmodel decorator use AccessType.WRITE,
# once permissions work
params = self._decodeParams(params)
self.requireParams(['userIds'], params)
creatorUser = self.getCurrentUser()
User().requireAdminStudy(creatorUser)
# Load all users before adding any, to ensure all are valid
if len(set(params['userIds'])) != len(params['userIds']):
raise ValidationException('Duplicate user IDs.', 'userIds')
annotatorUsers = [
User().load(userId, user=creatorUser, level=AccessType.READ, exc=True)
for userId in params['userIds']
]
# Existing duplicate Annotators are tricky to check for, because it's
# possible to have a Study with multiple annotator Users (each with a
# sub-Folder), but with no Images yet, and thus no Annotation (Items)
# inside yet
duplicateAnnotatorFolders = Folder().find({
'parentId': study['_id'],
'meta.userId': {'$in': [annotatorUser['_id'] for annotatorUser in annotatorUsers]}
})
if duplicateAnnotatorFolders.count():
# Just list the first duplicate
duplicateAnnotatorFolder = next(iter(duplicateAnnotatorFolders))
raise ValidationException('Annotator user "%s" is already part of the study.' %
duplicateAnnotatorFolder['meta']['userId'])
# Look up images only once for efficiency
images = Study().getImages(study)
for annotatorUser in annotatorUsers:
Study().addAnnotator(study, annotatorUser, creatorUser, images)
return self.getStudy(id=study['_id'], params={})
def validate(self, doc):
doc['name'] = doc['name'].strip()
if doc.get('description'):
doc['description'] = doc['description'].strip()
if doc.get('instructions'):
doc['instructions'] = doc['instructions'].strip()
if not doc['name']:
raise ValidationException('Challenge name must not be empty.',
'name')
# Ensure unique name for the collection
q = {'name': doc['name']}
if '_id' in doc:
q['_id'] = {'$ne': doc['_id']}
duplicate = self.findOne(q, fields=['_id'])
if duplicate is not None:
raise ValidationException(
'A challenge with that name already '
'exists.', 'name')
if doc.get('startDate'):
doc['startDate'] = validateDate(doc['startDate'], 'startDate')
if doc.get('endDate'):
doc['endDate'] = validateDate(doc['endDate'], 'endDate')
# Check that dates are in a sensible order
if doc.get('startDate') and doc.get('endDate'):
if doc['startDate'] >= doc['endDate']:
raise ValidationException('Invalid start and end dates.',
field='startDate')
return doc
def _validateParam(param):
if param.channel == 'input' and param.typ not in _SLICER_TO_GIRDER_WORKER_INPUT_TYPE_MAP:
raise ValidationException(
'Input parameter type %s is currently not supported.' % param.typ)
if param.channel == 'output' and param.typ not in _SLICER_TO_GIRDER_WORKER_OUTPUT_TYPE_MAP:
raise ValidationException(
'Output parameter type %s is currently not supported.' % param.typ)
def _validateFeatureId(self, name):
if not isinstance(name, six.string_types):
raise ValidationException('ID must be a string')
if not name:
raise ValidationException('ID must not be empty')
if not re.match('^[a-z0-9-_/]+$', name):
raise ValidationException(
'ID may only contain lower-case letters, numbers, '
'dashes, underscores, and forward-slashes')
def validate(self, doc, **kwargs):
if not doc.get('parentCollection') == 'folder':
raise ValidationException(
'A Slide model must be a child of a folder')
super().validate(doc, **kwargs)
case = Case().load(doc['parentId'], force=True)
if not case or self.getTCGAType(case) != 'case':
raise ValidationException(
'A Slide model must be a child of a case')
def validateCullingFrequency(doc):
if not doc['value']:
raise ValidationException(
'Culling frequency must not be empty.', 'value')
try:
float(doc['value'])
except ValueError:
raise ValidationException(
'Culling frequency must float.', 'value')
def validateProvidersEnabled(doc):
if not isinstance(doc['value'], (list, tuple)):
raise ValidationException('The enabled providers must be a list.', 'value')
for provider in doc['value']:
if not isinstance(provider, dict):
raise ValidationException('Providers should be JSON objects.')
if 'url' not in provider or 'name' not in provider:
raise ValidationException('Providers must have a "name" and "url" field.')
def validateZenodoExtraHosts(doc):
if not doc['value']:
doc['value'] = defaultZenodoExtraHosts()
if not isinstance(doc['value'], list):
raise ValidationException('Zenodo extra hosts setting must be a list.',
'value')
for url in doc['value']:
if not validators.url(url):
raise ValidationException('Invalid URL in Zenodo extra hosts',
'value')
def validateDataverseExtraHosts(doc):
if not doc['value']:
doc['value'] = defaultDataverseExtraHosts()
if not isinstance(doc['value'], list):
raise ValidationException(
'Dataverse extra hosts setting must be a list.', 'value')
for url in doc['value']:
if not validators.domain(url):
raise ValidationException(
'Invalid domain in Dataverse extra hosts', 'value')
def _parseFilter(self, filterParam):
if isinstance(filterParam, six.string_types):
try:
filterParam = json.loads(filterParam)
except ValueError as e:
raise ValidationException('Could not decode JSON: %s' % str(e), 'filter')
try:
return querylang.astToMongo(filterParam)
except (TypeError, ValueError) as e:
raise ValidationException('Could not parse filter:' % str(e), 'filter')
def validateInfo(doc):
"""
Makes sure the database name is valid.
"""
if not doc.get('db', ''):
raise ValidationException('Database name must not be empty.', 'db')
if '.' in doc['db'] or ' ' in doc['db']:
raise ValidationException('Database name cannot contain spaces'
' or periods.', 'db')
return doc
def _validateLogo(doc):
try:
logoFile = File().load(doc['value'], level=AccessType.READ, user=None, exc=True)
except ValidationException as e:
# Invalid ObjectId, or non-existent document
raise ValidationException(e.message, 'value')
except AccessException as e:
raise ValidationException('Logo must be publicly readable', 'value')
# Store this field natively as an ObjectId
doc['value'] = logoFile['_id']
