def setTimestamp(self, id, type, created, updated):
user = self.getCurrentUser()
model = self._getResourceModel(type)
doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True)
if created is not None:
if 'created' not in doc:
raise RestException('Resource has no "created" field.')
doc['created'] = parseTimestamp(created)
if updated is not None:
if 'updated' not in doc:
raise RestException('Resource has no "updated" field.')
doc['updated'] = parseTimestamp(updated)
return model.filter(model.save(doc), user=user)
def setTimestamp(self, id, type, created, updated):
user = self.getCurrentUser()
model = self._getResourceModel(type)
doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True)
if created is not None:
if 'created' not in doc:
raise RestException('Resource has no "created" field.')
doc['created'] = parseTimestamp(created)
if updated is not None:
if 'updated' not in doc:
raise RestException('Resource has no "updated" field.')
doc['updated'] = parseTimestamp(updated)
return model.filter(model.save(doc), user=user)
def setTimestamp(self, id, params):
user = self.getCurrentUser()
model = self._getResourceModel(params['type'])
doc = model.load(id=id, user=user, level=AccessType.WRITE)
if not doc:
raise RestException('Resource not found.')
if 'created' in params:
if 'created' not in doc:
raise RestException('Resource has no "created" field.')
doc['created'] = parseTimestamp(params['created'])
if 'updated' in params:
if 'updated' not in doc:
raise RestException('Resource has no "updated" field.')
doc['updated'] = parseTimestamp(params['updated'])
return model.save(doc)
def setTimestamp(self, id, params):
user = self.getCurrentUser()
model = self._getResourceModel(params['type'])
doc = model.load(id=id, user=user, level=AccessType.WRITE)
if not doc:
raise RestException('Resource not found.')
if 'created' in params:
if 'created' not in doc:
raise RestException('Resource has no "created" field.')
doc['created'] = parseTimestamp(params['created'])
if 'updated' in params:
if 'updated' not in doc:
raise RestException('Resource has no "updated" field.')
doc['updated'] = parseTimestamp(params['updated'])
return model.save(doc)
def create(self, **kwargs):
model = {}
for prop in self.create_props:
prop_value = kwargs.get(prop['name'], prop.get('default'))
if prop_value is not None:
if prop.get('type') == 'file':
file = File().load(prop_value,
user=getCurrentUser(),
level=AccessType.READ)
if file is None:
raise ValidationException('File doesn\'t exists: %s' %
prop_value)
if not isinstance(prop_value, ObjectId):
prop_value = ObjectId(prop_value)
elif prop.get('type') == ObjectId:
if isinstance(prop_value, list):
prop_value = [ObjectId(x) for x in prop_value]
else:
prop_value = ObjectId(prop_value)
elif prop.get('type') == 'timestamp':
prop_value = parseTimestamp(prop_value)
model[prop['name']] = prop_value
self.setPublic(model, public=kwargs.get('public', False))
user = kwargs.get('user')
self.setUserAccess(model, user=user, level=AccessType.ADMIN)
model['owner'] = user['_id']
if edp_group() is not None:
self.setGroupAccess(model, edp_group(), AccessType.ADMIN)
saved_model = self.save(model)
# Now spawn thumbnail jobs if the model contains any image
for prop in self.create_props:
prop_value = kwargs.get(prop['name'], prop.get('default'))
if prop_value is not None and prop.get('type') == 'file':
file = File().load(prop_value,
user=getCurrentUser(),
level=AccessType.READ)
mime_type = file.get('mimeType', '')
if mime_type is not None and mime_type.startswith('image/'):
self._create_thumbnail(file, saved_model, prop['name'],
user)
return saved_model
def testUploadDataset(self):
File = self.model('file')
Folder = self.model('folder')
Group = self.model('group')
Upload = self.model('upload')
User = self.model('user', 'isic_archive')
# Create a reviewer user that will receive notification emails
resp = self.request(path='/user',
method='POST',
params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'reviewer',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
reviewerUser = User.findOne({'login': '******'})
reviewersGroup = Group.findOne({'name': 'Dataset QC Reviewers'})
Group.addUser(reviewersGroup, reviewerUser, level=AccessType.READ)
# Create an uploader user
resp = self.request(path='/user',
method='POST',
params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'uploader',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
uploaderUser = User.findOne({'login': '******'})
contributorsGroup = Group.findOne({'name': 'Dataset Contributors'})
Group.addUser(contributorsGroup, uploaderUser, level=AccessType.READ)
# Create and upload two ZIP files of images
publicDataset = self._uploadDataset(
uploaderUser=uploaderUser,
zipName='test_zip_1',
zipContentNames=[
'test_1_small_1.jpg', 'test_1_small_2.jpg',
'test_1_large_1.jpg'
],
datasetName='test_dataset_1',
datasetDescription='A public test dataset')
privateDataset = self._uploadDataset(
uploaderUser=uploaderUser,
zipName='test_zip_2',
zipContentNames=['test_1_small_3.jpg', 'test_1_large_2.jpg'],
datasetName='test_dataset_2',
datasetDescription='A private test dataset')
# Ensure that ordinary users aren't getting review tasks
resp = self.request(path='/task/me/review', method='GET')
self.assertStatus(resp, 401)
resp = self.request(path='/task/me/review',
method='GET',
user=uploaderUser)
self.assertStatus(resp, 403)
# Ensure that reviewer users are getting tasks
resp = self.request(path='/task/me/review',
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
reviewTasks = resp.json
self.assertEqual(len(reviewTasks), 2)
self.assertIn(
{
'dataset': {
'_id': str(publicDataset['_id']),
'name': publicDataset['name']
},
'count': 3
}, reviewTasks)
self.assertIn(
{
'dataset': {
'_id': str(privateDataset['_id']),
'name': privateDataset['name']
},
'count': 2
}, reviewTasks)
# Ensure that review task redirects are working
resp = self.request(path='/task/me/review/redirect',
method='GET',
user=reviewerUser)
self.assertStatus(resp, 400)
for reviewTask in reviewTasks:
reviewId = reviewTask['dataset']['_id']
resp = self.request(path='/task/me/review/redirect',
method='GET',
params={'datasetId': reviewId},
user=reviewerUser,
isJson=False)
self.assertStatus(resp, 307)
self.assertDictContainsSubset(
{'Location': '/markup/gallery#/qc/%s' % reviewId},
resp.headers)
# Accept all images
resp = self.request(path='/dataset/%s/review' % publicDataset['_id'],
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 3)
imageIds = [image['_id'] for image in resp.json]
resp = self.request(path='/dataset/%s/review' % publicDataset['_id'],
method='POST',
user=reviewerUser,
params={
'accepted': json.dumps(imageIds),
'flagged': []
})
self.assertStatusOk(resp)
# Test metadata registration
resp = self.request(path='/folder',
method='POST',
user=uploaderUser,
params={
'parentType': 'user',
'parentId': str(uploaderUser['_id']),
'name': 'test_1_metadata_folder'
})
self.assertStatusOk(resp)
uploadCsvFolder = Folder.load(resp.json['_id'], force=True)
# Upload the CSV metadata file
csvPath = os.path.join(self.testDataDir, 'test_1_metadata.csv')
with open(csvPath, 'rb') as csvStream:
metadataFile = Upload.uploadFromFile(obj=csvStream,
size=os.path.getsize(csvPath),
name='test_1_metadata.csv',
parentType='folder',
parent=uploadCsvFolder,
user=uploaderUser,
mimeType='text/csv')
# Attempt to register metadata as invalid users
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
params={'metadataFileId': metadataFile['_id']})
self.assertStatus(resp, 401)
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
user=reviewerUser,
params={'metadataFileId': metadataFile['_id']})
self.assertStatus(resp, 403)
# Attempt to register metadata with invalid parameters
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
user=uploaderUser)
self.assertStatus(resp, 400)
self.assertIn('required', resp.json['message'].lower())
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
user=uploaderUser,
params={'metadataFileId': 'bad_id'})
self.assertStatus(resp, 400)
self.assertIn('invalid', resp.json['message'].lower())
resp = self.request(
path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
user=uploaderUser,
params={
# TODO: find a cleaner way to pass a file with the wrong format
'metadataFileId':
File.findOne({'mimeType': 'application/zip'})['_id'],
})
self.assertStatus(resp, 400)
self.assertIn('format', resp.json['message'].lower())
# Attempt to list registered metadata as invalid users
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET')
self.assertStatus(resp, 401)
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET',
user=uploaderUser)
self.assertStatus(resp, 403)
# List (empty) registered metadata
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
# Register metadata with dataset
self.assertNoMail()
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='POST',
user=uploaderUser,
params={'metadataFileId': metadataFile['_id']})
self.assertStatusOk(resp)
# Reviewer user should receive email
self.assertMails(count=1)
# List registered metadata
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
user=reviewerUser)
self.assertStatusOk(resp)
self.assertIsInstance(resp.json, list)
self.assertEqual(len(resp.json), 1)
# Check the 'time' field separately, as we don't know what it will be
self.assertIn('time', resp.json[0])
self.assertLess(parseTimestamp(resp.json[0]['time']),
datetime.datetime.utcnow())
self.assertDictEqual(
{
'file': {
'_id': str(metadataFile['_id']),
'name': metadataFile['name']
},
'user': {
'_id': str(uploaderUser['_id']),
'name': User.obfuscatedName(uploaderUser)
},
# This is actually checked above
'time': resp.json[0]['time']
},
resp.json[0])
# Test applying metadata
resp = self.request(path='/dataset/%s/metadata/%s' %
(publicDataset['_id'], metadataFile['_id']),
method='POST',
user=uploaderUser,
params={'save': False})
self.assertStatusOk(resp)
self.assertIn('errors', resp.json)
self.assertIn('warnings', resp.json)
self.assertEqual(resp.json['errors'], [{
'description':
'on CSV row 5: values [u\'solar lentigo\', False] for fields [\'diagnosis\', '
'\'melanocytic\'] are inconsistent'
}])
self.assertEqual(resp.json['warnings'], [{
'description':
'on CSV row 4: no images found that match u\'filename\': u\'test_1_small_3.jpg\''
}, {
'description':
'on CSV row 6: no images found that match u\'filename\': u\'test_1_large_2.jpg\''
}, {
'description':
'unrecognized field u\'age_approx\' will be added to unstructured metadata'
}, {
'description':
'unrecognized field u\'isic_source_name\' will be added to unstructured metadata'
}])
def testFeatureset(self):
Featureset = self.model('featureset', 'isic_archive')
Group = self.model('group')
Study = self.model('study', 'isic_archive')
User = self.model('user', 'isic_archive')
# Create a basic user
resp = self.request(path='/user', method='POST', params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'basic',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
basicUser = User.findOne({'login': '******'})
# Create a study admin user
resp = self.request(path='/user', method='POST', params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'study admin',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
studyAdminUser = User.findOne({'login': '******'})
studyAdminsGroup = Group.findOne({'name': 'Study Administrators'})
Group.addUser(studyAdminsGroup, studyAdminUser, level=AccessType.READ)
basicFeaturesetParams = {
'name': ' Basic ',
'version': '1.0',
'globalFeatures': [
{
'id': 'quality',
'name': ['Quality'],
'options': [
{
'id': 'acceptable',
'name': 'Acceptable'
},
{
'id': 'unacceptable',
'name': 'Unacceptable'
}
],
'type': 'radio'
},
{
'id': 'diagnosis',
'name': ['Diagnosis'],
'options': [
{
'id': 'benign',
'name': 'Benign'
},
{
'id': 'indeterminate',
'name': 'Indeterminate'
},
{
'id': 'malignant',
'name': 'Malignant'
}
],
'type': 'radio'
}
],
'localFeatures': [
{
'id': 'lesion',
'name': ['Lesion'],
'type': 'superpixel'
},
{
'id': 'skin',
'name': ['Normal Skin'],
'type': 'superpixel'
}
]
}
# Try to create a featureset as anonymous
resp = self.request(
path='/featureset', method='POST', params=basicFeaturesetParams)
self.assertStatus(resp, 401)
# Try to create a featureset without privileges
resp = self.request(
path='/featureset', method='POST', user=basicUser,
params=basicFeaturesetParams)
self.assertStatus(resp, 403)
# Try to create a featureset with an empty name
tempFeaturesetParams = basicFeaturesetParams.copy()
tempFeaturesetParams['name'] = ''
resp = self.request(
path='/featureset', method='POST', user=studyAdminUser,
type='application/json', body=json.dumps(tempFeaturesetParams))
self.assertStatus(resp, 400)
# Try to create a featureset with an invalid version
tempFeaturesetParams = basicFeaturesetParams.copy()
tempFeaturesetParams['version'] = 'foo'
resp = self.request(
path='/featureset', method='POST', user=studyAdminUser,
type='application/json', body=json.dumps(tempFeaturesetParams))
self.assertStatus(resp, 400)
# TODO: Try to create a featureset with invalid features
# List all (nonexistent) featuresets
resp = self.request(
path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
# Create a valid featureset
resp = self.request(
path='/featureset', method='POST', user=studyAdminUser,
type='application/json', body=json.dumps(basicFeaturesetParams))
self.assertStatusOk(resp)
# Lookup the database entry, for access to non-deterministic details
# like '_id' and 'created'
basicFeatureset = Featureset.load(resp.json['_id'])
# Check the list of all featuresets
resp = self.request(
path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertIsInstance(resp.json, list)
self.assertEqual(len(resp.json), 1)
self.assertDictEqual({
'_id': str(basicFeatureset['_id']),
'name': basicFeaturesetParams['name'].strip(),
'version': float(basicFeaturesetParams['version']),
}, resp.json[0])
# Check featureset details as anonymous
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='GET')
self.assertStatusOk(resp)
# Parse the "created" field first, to make comparison easier
self.assertIn('created', resp.json)
resp.json['created'] = parseTimestamp(resp.json['created'])
self.assertDictEqual({
'_id': str(basicFeatureset['_id']),
'_modelType': 'featureset',
'name': basicFeaturesetParams['name'].strip(),
'version': float(basicFeaturesetParams['version']),
'created': basicFeatureset['created'],
'creator': {
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser)
},
'globalFeatures': basicFeaturesetParams['globalFeatures'],
'localFeatures': basicFeaturesetParams['localFeatures']
}, resp.json)
# Ensure that normal users don't get private creator info
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='GET',
user=basicUser)
self.assertStatusOk(resp)
self.assertDictEqual({
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser)
}, resp.json['creator'])
# Ensure that study admin users do get private creator info
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='GET',
user=studyAdminUser)
self.assertStatusOk(resp)
self.assertDictEqual({
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser),
'firstName': studyAdminUser['firstName'],
'lastName': studyAdminUser['lastName'],
'login': studyAdminUser['login']
}, resp.json['creator'])
# Try to delete a featureset as anonymous
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='DELETE')
self.assertStatus(resp, 401)
# Try to delete a featureset without privileges
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='DELETE',
user=basicUser)
self.assertStatus(resp, 403)
# Try to delete a featureset being used by a study
resp = self.request(
path='/study', method='POST', user=studyAdminUser,
type='application/json', body=json.dumps({
'name': 'Test Study',
'featuresetId': str(basicFeatureset['_id']),
'userIds': [],
'imageIds': []
}))
self.assertStatusOk(resp)
testStudy = Study.load(resp.json['_id'], force=True)
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='DELETE',
user=studyAdminUser)
self.assertStatus(resp, 409)
resp = self.request(
path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
# TODO: Use the Study API, once it exists
Study.remove(testStudy)
# Delete an unused featureset
resp = self.request(
path='/featureset/%s' % basicFeatureset['_id'], method='DELETE',
user=studyAdminUser, isJson=False)
self.assertStatus(resp, 204)
resp = self.request(
path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
def testFeatureset(self):
Featureset = self.model('featureset', 'isic_archive')
Group = self.model('group')
Study = self.model('study', 'isic_archive')
User = self.model('user', 'isic_archive')
# Create a basic user
resp = self.request(path='/user',
method='POST',
params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'basic',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
basicUser = User.findOne({'login': '******'})
# Create a study admin user
resp = self.request(path='/user',
method='POST',
params={
'email': '*****@*****.**',
'login': '******',
'firstName': 'study admin',
'lastName': 'user',
'password': '******'
})
self.assertStatusOk(resp)
studyAdminUser = User.findOne({'login': '******'})
studyAdminsGroup = Group.findOne({'name': 'Study Administrators'})
Group.addUser(studyAdminsGroup, studyAdminUser, level=AccessType.READ)
basicFeaturesetParams = {
'name':
' Basic ',
'version':
'1.0',
'globalFeatures': [{
'id':
'quality',
'name': ['Quality'],
'options': [{
'id': 'acceptable',
'name': 'Acceptable'
}, {
'id': 'unacceptable',
'name': 'Unacceptable'
}],
'type':
'radio'
}, {
'id':
'diagnosis',
'name': ['Diagnosis'],
'options': [{
'id': 'benign',
'name': 'Benign'
}, {
'id': 'indeterminate',
'name': 'Indeterminate'
}, {
'id': 'malignant',
'name': 'Malignant'
}],
'type':
'radio'
}],
'localFeatures': [{
'id': 'lesion',
'name': ['Lesion'],
'type': 'superpixel'
}, {
'id': 'skin',
'name': ['Normal Skin'],
'type': 'superpixel'
}]
}
# Try to create a featureset as anonymous
resp = self.request(path='/featureset',
method='POST',
params=basicFeaturesetParams)
self.assertStatus(resp, 401)
# Try to create a featureset without privileges
resp = self.request(path='/featureset',
method='POST',
user=basicUser,
params=basicFeaturesetParams)
self.assertStatus(resp, 403)
# Try to create a featureset with an empty name
tempFeaturesetParams = basicFeaturesetParams.copy()
tempFeaturesetParams['name'] = ''
resp = self.request(path='/featureset',
method='POST',
user=studyAdminUser,
type='application/json',
body=json.dumps(tempFeaturesetParams))
self.assertStatus(resp, 400)
# Try to create a featureset with an invalid version
tempFeaturesetParams = basicFeaturesetParams.copy()
tempFeaturesetParams['version'] = 'foo'
resp = self.request(path='/featureset',
method='POST',
user=studyAdminUser,
type='application/json',
body=json.dumps(tempFeaturesetParams))
self.assertStatus(resp, 400)
# TODO: Try to create a featureset with invalid features
# List all (nonexistent) featuresets
resp = self.request(path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
# Create a valid featureset
resp = self.request(path='/featureset',
method='POST',
user=studyAdminUser,
type='application/json',
body=json.dumps(basicFeaturesetParams))
self.assertStatusOk(resp)
# Lookup the database entry, for access to non-deterministic details
# like '_id' and 'created'
basicFeatureset = Featureset.load(resp.json['_id'])
# Check the list of all featuresets
resp = self.request(path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertIsInstance(resp.json, list)
self.assertEqual(len(resp.json), 1)
self.assertDictEqual(
{
'_id': str(basicFeatureset['_id']),
'name': basicFeaturesetParams['name'].strip(),
'version': float(basicFeaturesetParams['version']),
}, resp.json[0])
# Check featureset details as anonymous
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='GET')
self.assertStatusOk(resp)
# Parse the "created" field first, to make comparison easier
self.assertIn('created', resp.json)
resp.json['created'] = parseTimestamp(resp.json['created'])
self.assertDictEqual(
{
'_id': str(basicFeatureset['_id']),
'_modelType': 'featureset',
'name': basicFeaturesetParams['name'].strip(),
'version': float(basicFeaturesetParams['version']),
'created': basicFeatureset['created'],
'creator': {
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser)
},
'globalFeatures': basicFeaturesetParams['globalFeatures'],
'localFeatures': basicFeaturesetParams['localFeatures']
}, resp.json)
# Ensure that normal users don't get private creator info
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='GET',
user=basicUser)
self.assertStatusOk(resp)
self.assertDictEqual(
{
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser)
}, resp.json['creator'])
# Ensure that study admin users do get private creator info
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='GET',
user=studyAdminUser)
self.assertStatusOk(resp)
self.assertDictEqual(
{
'_id': str(studyAdminUser['_id']),
'name': User.obfuscatedName(studyAdminUser),
'firstName': studyAdminUser['firstName'],
'lastName': studyAdminUser['lastName'],
'login': studyAdminUser['login']
}, resp.json['creator'])
# Try to delete a featureset as anonymous
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='DELETE')
self.assertStatus(resp, 401)
# Try to delete a featureset without privileges
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='DELETE',
user=basicUser)
self.assertStatus(resp, 403)
# Try to delete a featureset being used by a study
resp = self.request(path='/study',
method='POST',
user=studyAdminUser,
type='application/json',
body=json.dumps({
'name':
'Test Study',
'featuresetId':
str(basicFeatureset['_id']),
'userIds': [],
'imageIds': []
}))
self.assertStatusOk(resp)
testStudy = Study.load(resp.json['_id'], force=True)
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='DELETE',
user=studyAdminUser)
self.assertStatus(resp, 409)
resp = self.request(path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
# TODO: Use the Study API, once it exists
Study.remove(testStudy)
# Delete an unused featureset
resp = self.request(path='/featureset/%s' % basicFeatureset['_id'],
method='DELETE',
user=studyAdminUser,
isJson=False)
self.assertStatus(resp, 204)
resp = self.request(path='/featureset', method='GET')
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
def testUploadDataset(self):
User = self.model('user', 'isic_archive')
# Create users
reviewerUser = self._createReviewerUser()
uploaderUser = self._createUploaderUser()
adminUser = self._createSiteAdminUser()
# Create and upload two ZIP files of images
publicDataset = self._uploadDataset(
uploaderUser=uploaderUser,
zipName='test_zip_1',
zipContentNames=[
'test_1_small_1.jpg', 'test_1_small_2.jpg',
'test_1_large_1.jpg'
],
datasetName='test_dataset_1',
datasetDescription='A public test dataset')
privateDataset = self._uploadDataset(
uploaderUser=uploaderUser,
zipName='test_zip_2',
zipContentNames=['test_1_small_3.jpg', 'test_1_large_2.jpg'],
datasetName='test_dataset_2',
datasetDescription='A private test dataset')
# Ensure that ordinary users aren't getting review tasks
resp = self.request(path='/task/me/review', method='GET')
self.assertStatus(resp, 401)
resp = self.request(path='/task/me/review',
method='GET',
user=uploaderUser)
self.assertStatus(resp, 403)
# Ensure that reviewer users are getting tasks
resp = self.request(path='/task/me/review',
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
reviewTasks = resp.json
self.assertEqual(len(reviewTasks), 2)
self.assertIn(
{
'dataset': {
'_id': str(publicDataset['_id']),
'name': publicDataset['name']
},
'count': 3
}, reviewTasks)
self.assertIn(
{
'dataset': {
'_id': str(privateDataset['_id']),
'name': privateDataset['name']
},
'count': 2
}, reviewTasks)
# Ensure that review task redirects are working
resp = self.request(path='/task/me/review/redirect',
method='GET',
user=reviewerUser)
self.assertStatus(resp, 400)
for reviewTask in reviewTasks:
reviewId = reviewTask['dataset']['_id']
resp = self.request(path='/task/me/review/redirect',
method='GET',
params={'datasetId': reviewId},
user=reviewerUser,
isJson=False)
self.assertStatus(resp, 307)
self.assertDictContainsSubset(
{'Location': '/#tasks/review/%s' % reviewId}, resp.headers)
# Accept all images
resp = self.request(path='/dataset/%s/review' % publicDataset['_id'],
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 3)
imageIds = [image['_id'] for image in resp.json]
resp = self.request(path='/dataset/%s/review' % publicDataset['_id'],
method='POST',
user=reviewerUser,
params={
'accepted': json.dumps(imageIds),
'flagged': []
})
self.assertStatusOk(resp)
# Attempt to register metadata as invalid users
csvPath = os.path.join(self.testDataDir, 'test_1_metadata.csv')
with open(csvPath, 'rb') as csvStream:
resp = self.request(path='/dataset/%s/metadata' %
publicDataset['_id'],
method='POST',
body=csvStream.read(),
type='text/csv',
params={'filename': 'test_1_metadata.csv'})
self.assertStatus(resp, 401)
# Attempt to register metadata with invalid parameters
with open(csvPath, 'rb') as csvStream:
resp = self.request(path='/dataset/%s/metadata' %
publicDataset['_id'],
method='POST',
body=csvStream.read(),
type='text/csv',
user=uploaderUser)
self.assertStatus(resp, 400)
self.assertIn('"filename" is required', resp.json['message'].lower())
with open(csvPath, 'rb') as csvStream:
resp = self.request(path='/dataset/%s/metadata' %
publicDataset['_id'],
method='POST',
body=csvStream.read(),
type='text/csv',
user=uploaderUser,
params={'filename': ' '})
self.assertStatus(resp, 400)
self.assertIn('filename must be specified',
resp.json['message'].lower())
# Attempt to list registered metadata as invalid users
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET')
self.assertStatus(resp, 401)
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET',
user=uploaderUser)
self.assertStatus(resp, 403)
# List (empty) registered metadata
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
method='GET',
user=reviewerUser)
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
# Register metadata with dataset
self.assertNoMail()
with open(csvPath, 'rb') as csvStream:
resp = self.request(path='/dataset/%s/metadata' %
publicDataset['_id'],
method='POST',
body=csvStream.read(),
type='text/csv',
isJson=False,
user=uploaderUser,
params={'filename': 'test_1_metadata.csv'})
self.assertStatusOk(resp)
# Reviewer user should receive email
self.assertMails(count=1)
# List registered metadata
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
user=reviewerUser)
self.assertStatusOk(resp)
self.assertIsInstance(resp.json, list)
self.assertEqual(len(resp.json), 1)
# Check file field
self.assertIn('file', resp.json[0])
self.assertIn('_id', resp.json[0]['file'])
self.assertIn('name', resp.json[0]['file'])
self.assertEqual('test_1_metadata.csv', resp.json[0]['file']['name'])
self.assertIn('user', resp.json[0])
# Check user field
self.assertDictEqual(
{
'_id': str(uploaderUser['_id']),
'name': User.obfuscatedName(uploaderUser)
}, resp.json[0]['user'])
# Check time field
self.assertIn('time', resp.json[0])
self.assertLess(parseTimestamp(resp.json[0]['time']),
datetime.datetime.utcnow())
metadataFileId = resp.json[0]['file']['_id']
# Test downloading metadata as invalid users
resp = self.request(path='/dataset/%s/metadata/%s/download' %
(publicDataset['_id'], metadataFileId),
method='GET',
isJson=False)
self.assertStatus(resp, 401)
resp = self.request(path='/dataset/%s/metadata/%s/download' %
(publicDataset['_id'], metadataFileId),
method='GET',
user=uploaderUser,
isJson=False)
self.assertStatus(resp, 403)
# Test downloading metadata
resp = self.request(path='/dataset/%s/metadata/%s/download' %
(publicDataset['_id'], metadataFileId),
method='GET',
user=reviewerUser,
isJson=False)
with open(csvPath, 'rb') as csvStream:
self.assertEqual(csvStream.read(), self.getBody(resp))
# Test applying metadata
resp = self.request(path='/dataset/%s/metadata/%s/apply' %
(publicDataset['_id'], metadataFileId),
method='POST',
user=uploaderUser,
params={'save': False})
self.assertStatus(resp, 403)
resp = self.request(path='/dataset/%s/metadata/%s/apply' %
(publicDataset['_id'], metadataFileId),
method='POST',
user=reviewerUser,
params={'save': False})
self.assertStatusOk(resp)
self.assertIn('errors', resp.json)
self.assertIn('warnings', resp.json)
self.assertEqual(0, len(resp.json['errors']))
self.assertEqual(resp.json['warnings'], [{
'description':
'on CSV row 4: no images found that match u\'filename\': u\'test_1_small_3.jpg\''
}, {
'description':
'on CSV row 6: no images found that match u\'filename\': u\'test_1_large_2.jpg\''
}, {
'description':
'unrecognized field u\'age_approx\' will be added to unstructured metadata'
}, {
'description':
'unrecognized field u\'isic_source_name\' will be added to unstructured metadata'
}])
# Test removing metadata as site admin
resp = self.request(path='/dataset/%s/metadata/%s' %
(publicDataset['_id'], metadataFileId),
method='DELETE',
user=adminUser,
isJson=False)
self.assertStatus(resp, 204)
resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'],
user=reviewerUser)
self.assertStatusOk(resp)
self.assertIsInstance(resp.json, list)
self.assertEqual(len(resp.json), 0)