def setTimestamp(self, id, type, created, updated): user = self.getCurrentUser() model = self._getResourceModel(type) doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True) if created is not None: if 'created' not in doc: raise RestException('Resource has no "created" field.') doc['created'] = parseTimestamp(created) if updated is not None: if 'updated' not in doc: raise RestException('Resource has no "updated" field.') doc['updated'] = parseTimestamp(updated) return model.filter(model.save(doc), user=user)
def setTimestamp(self, id, type, created, updated): user = self.getCurrentUser() model = self._getResourceModel(type) doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True) if created is not None: if 'created' not in doc: raise RestException('Resource has no "created" field.') doc['created'] = parseTimestamp(created) if updated is not None: if 'updated' not in doc: raise RestException('Resource has no "updated" field.') doc['updated'] = parseTimestamp(updated) return model.filter(model.save(doc), user=user)
def setTimestamp(self, id, params): user = self.getCurrentUser() model = self._getResourceModel(params['type']) doc = model.load(id=id, user=user, level=AccessType.WRITE) if not doc: raise RestException('Resource not found.') if 'created' in params: if 'created' not in doc: raise RestException('Resource has no "created" field.') doc['created'] = parseTimestamp(params['created']) if 'updated' in params: if 'updated' not in doc: raise RestException('Resource has no "updated" field.') doc['updated'] = parseTimestamp(params['updated']) return model.save(doc)
def setTimestamp(self, id, params): user = self.getCurrentUser() model = self._getResourceModel(params['type']) doc = model.load(id=id, user=user, level=AccessType.WRITE) if not doc: raise RestException('Resource not found.') if 'created' in params: if 'created' not in doc: raise RestException('Resource has no "created" field.') doc['created'] = parseTimestamp(params['created']) if 'updated' in params: if 'updated' not in doc: raise RestException('Resource has no "updated" field.') doc['updated'] = parseTimestamp(params['updated']) return model.save(doc)
def create(self, **kwargs): model = {} for prop in self.create_props: prop_value = kwargs.get(prop['name'], prop.get('default')) if prop_value is not None: if prop.get('type') == 'file': file = File().load(prop_value, user=getCurrentUser(), level=AccessType.READ) if file is None: raise ValidationException('File doesn\'t exists: %s' % prop_value) if not isinstance(prop_value, ObjectId): prop_value = ObjectId(prop_value) elif prop.get('type') == ObjectId: if isinstance(prop_value, list): prop_value = [ObjectId(x) for x in prop_value] else: prop_value = ObjectId(prop_value) elif prop.get('type') == 'timestamp': prop_value = parseTimestamp(prop_value) model[prop['name']] = prop_value self.setPublic(model, public=kwargs.get('public', False)) user = kwargs.get('user') self.setUserAccess(model, user=user, level=AccessType.ADMIN) model['owner'] = user['_id'] if edp_group() is not None: self.setGroupAccess(model, edp_group(), AccessType.ADMIN) saved_model = self.save(model) # Now spawn thumbnail jobs if the model contains any image for prop in self.create_props: prop_value = kwargs.get(prop['name'], prop.get('default')) if prop_value is not None and prop.get('type') == 'file': file = File().load(prop_value, user=getCurrentUser(), level=AccessType.READ) mime_type = file.get('mimeType', '') if mime_type is not None and mime_type.startswith('image/'): self._create_thumbnail(file, saved_model, prop['name'], user) return saved_model
def testUploadDataset(self): File = self.model('file') Folder = self.model('folder') Group = self.model('group') Upload = self.model('upload') User = self.model('user', 'isic_archive') # Create a reviewer user that will receive notification emails resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'reviewer', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) reviewerUser = User.findOne({'login': '******'}) reviewersGroup = Group.findOne({'name': 'Dataset QC Reviewers'}) Group.addUser(reviewersGroup, reviewerUser, level=AccessType.READ) # Create an uploader user resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'uploader', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) uploaderUser = User.findOne({'login': '******'}) contributorsGroup = Group.findOne({'name': 'Dataset Contributors'}) Group.addUser(contributorsGroup, uploaderUser, level=AccessType.READ) # Create and upload two ZIP files of images publicDataset = self._uploadDataset( uploaderUser=uploaderUser, zipName='test_zip_1', zipContentNames=[ 'test_1_small_1.jpg', 'test_1_small_2.jpg', 'test_1_large_1.jpg' ], datasetName='test_dataset_1', datasetDescription='A public test dataset') privateDataset = self._uploadDataset( uploaderUser=uploaderUser, zipName='test_zip_2', zipContentNames=['test_1_small_3.jpg', 'test_1_large_2.jpg'], datasetName='test_dataset_2', datasetDescription='A private test dataset') # Ensure that ordinary users aren't getting review tasks resp = self.request(path='/task/me/review', method='GET') self.assertStatus(resp, 401) resp = self.request(path='/task/me/review', method='GET', user=uploaderUser) self.assertStatus(resp, 403) # Ensure that reviewer users are getting tasks resp = self.request(path='/task/me/review', method='GET', user=reviewerUser) self.assertStatusOk(resp) reviewTasks = resp.json self.assertEqual(len(reviewTasks), 2) self.assertIn( { 'dataset': { '_id': str(publicDataset['_id']), 'name': publicDataset['name'] }, 'count': 3 }, reviewTasks) self.assertIn( { 'dataset': { '_id': str(privateDataset['_id']), 'name': privateDataset['name'] }, 'count': 2 }, reviewTasks) # Ensure that review task redirects are working resp = self.request(path='/task/me/review/redirect', method='GET', user=reviewerUser) self.assertStatus(resp, 400) for reviewTask in reviewTasks: reviewId = reviewTask['dataset']['_id'] resp = self.request(path='/task/me/review/redirect', method='GET', params={'datasetId': reviewId}, user=reviewerUser, isJson=False) self.assertStatus(resp, 307) self.assertDictContainsSubset( {'Location': '/markup/gallery#/qc/%s' % reviewId}, resp.headers) # Accept all images resp = self.request(path='/dataset/%s/review' % publicDataset['_id'], method='GET', user=reviewerUser) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 3) imageIds = [image['_id'] for image in resp.json] resp = self.request(path='/dataset/%s/review' % publicDataset['_id'], method='POST', user=reviewerUser, params={ 'accepted': json.dumps(imageIds), 'flagged': [] }) self.assertStatusOk(resp) # Test metadata registration resp = self.request(path='/folder', method='POST', user=uploaderUser, params={ 'parentType': 'user', 'parentId': str(uploaderUser['_id']), 'name': 'test_1_metadata_folder' }) self.assertStatusOk(resp) uploadCsvFolder = Folder.load(resp.json['_id'], force=True) # Upload the CSV metadata file csvPath = os.path.join(self.testDataDir, 'test_1_metadata.csv') with open(csvPath, 'rb') as csvStream: metadataFile = Upload.uploadFromFile(obj=csvStream, size=os.path.getsize(csvPath), name='test_1_metadata.csv', parentType='folder', parent=uploadCsvFolder, user=uploaderUser, mimeType='text/csv') # Attempt to register metadata as invalid users resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', params={'metadataFileId': metadataFile['_id']}) self.assertStatus(resp, 401) resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', user=reviewerUser, params={'metadataFileId': metadataFile['_id']}) self.assertStatus(resp, 403) # Attempt to register metadata with invalid parameters resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', user=uploaderUser) self.assertStatus(resp, 400) self.assertIn('required', resp.json['message'].lower()) resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', user=uploaderUser, params={'metadataFileId': 'bad_id'}) self.assertStatus(resp, 400) self.assertIn('invalid', resp.json['message'].lower()) resp = self.request( path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', user=uploaderUser, params={ # TODO: find a cleaner way to pass a file with the wrong format 'metadataFileId': File.findOne({'mimeType': 'application/zip'})['_id'], }) self.assertStatus(resp, 400) self.assertIn('format', resp.json['message'].lower()) # Attempt to list registered metadata as invalid users resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET') self.assertStatus(resp, 401) resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET', user=uploaderUser) self.assertStatus(resp, 403) # List (empty) registered metadata resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET', user=reviewerUser) self.assertStatusOk(resp) self.assertEqual(resp.json, []) # Register metadata with dataset self.assertNoMail() resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', user=uploaderUser, params={'metadataFileId': metadataFile['_id']}) self.assertStatusOk(resp) # Reviewer user should receive email self.assertMails(count=1) # List registered metadata resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], user=reviewerUser) self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 1) # Check the 'time' field separately, as we don't know what it will be self.assertIn('time', resp.json[0]) self.assertLess(parseTimestamp(resp.json[0]['time']), datetime.datetime.utcnow()) self.assertDictEqual( { 'file': { '_id': str(metadataFile['_id']), 'name': metadataFile['name'] }, 'user': { '_id': str(uploaderUser['_id']), 'name': User.obfuscatedName(uploaderUser) }, # This is actually checked above 'time': resp.json[0]['time'] }, resp.json[0]) # Test applying metadata resp = self.request(path='/dataset/%s/metadata/%s' % (publicDataset['_id'], metadataFile['_id']), method='POST', user=uploaderUser, params={'save': False}) self.assertStatusOk(resp) self.assertIn('errors', resp.json) self.assertIn('warnings', resp.json) self.assertEqual(resp.json['errors'], [{ 'description': 'on CSV row 5: values [u\'solar lentigo\', False] for fields [\'diagnosis\', ' '\'melanocytic\'] are inconsistent' }]) self.assertEqual(resp.json['warnings'], [{ 'description': 'on CSV row 4: no images found that match u\'filename\': u\'test_1_small_3.jpg\'' }, { 'description': 'on CSV row 6: no images found that match u\'filename\': u\'test_1_large_2.jpg\'' }, { 'description': 'unrecognized field u\'age_approx\' will be added to unstructured metadata' }, { 'description': 'unrecognized field u\'isic_source_name\' will be added to unstructured metadata' }])
def testFeatureset(self): Featureset = self.model('featureset', 'isic_archive') Group = self.model('group') Study = self.model('study', 'isic_archive') User = self.model('user', 'isic_archive') # Create a basic user resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'basic', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) basicUser = User.findOne({'login': '******'}) # Create a study admin user resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'study admin', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) studyAdminUser = User.findOne({'login': '******'}) studyAdminsGroup = Group.findOne({'name': 'Study Administrators'}) Group.addUser(studyAdminsGroup, studyAdminUser, level=AccessType.READ) basicFeaturesetParams = { 'name': ' Basic ', 'version': '1.0', 'globalFeatures': [ { 'id': 'quality', 'name': ['Quality'], 'options': [ { 'id': 'acceptable', 'name': 'Acceptable' }, { 'id': 'unacceptable', 'name': 'Unacceptable' } ], 'type': 'radio' }, { 'id': 'diagnosis', 'name': ['Diagnosis'], 'options': [ { 'id': 'benign', 'name': 'Benign' }, { 'id': 'indeterminate', 'name': 'Indeterminate' }, { 'id': 'malignant', 'name': 'Malignant' } ], 'type': 'radio' } ], 'localFeatures': [ { 'id': 'lesion', 'name': ['Lesion'], 'type': 'superpixel' }, { 'id': 'skin', 'name': ['Normal Skin'], 'type': 'superpixel' } ] } # Try to create a featureset as anonymous resp = self.request( path='/featureset', method='POST', params=basicFeaturesetParams) self.assertStatus(resp, 401) # Try to create a featureset without privileges resp = self.request( path='/featureset', method='POST', user=basicUser, params=basicFeaturesetParams) self.assertStatus(resp, 403) # Try to create a featureset with an empty name tempFeaturesetParams = basicFeaturesetParams.copy() tempFeaturesetParams['name'] = '' resp = self.request( path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(tempFeaturesetParams)) self.assertStatus(resp, 400) # Try to create a featureset with an invalid version tempFeaturesetParams = basicFeaturesetParams.copy() tempFeaturesetParams['version'] = 'foo' resp = self.request( path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(tempFeaturesetParams)) self.assertStatus(resp, 400) # TODO: Try to create a featureset with invalid features # List all (nonexistent) featuresets resp = self.request( path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(resp.json, []) # Create a valid featureset resp = self.request( path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(basicFeaturesetParams)) self.assertStatusOk(resp) # Lookup the database entry, for access to non-deterministic details # like '_id' and 'created' basicFeatureset = Featureset.load(resp.json['_id']) # Check the list of all featuresets resp = self.request( path='/featureset', method='GET') self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 1) self.assertDictEqual({ '_id': str(basicFeatureset['_id']), 'name': basicFeaturesetParams['name'].strip(), 'version': float(basicFeaturesetParams['version']), }, resp.json[0]) # Check featureset details as anonymous resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='GET') self.assertStatusOk(resp) # Parse the "created" field first, to make comparison easier self.assertIn('created', resp.json) resp.json['created'] = parseTimestamp(resp.json['created']) self.assertDictEqual({ '_id': str(basicFeatureset['_id']), '_modelType': 'featureset', 'name': basicFeaturesetParams['name'].strip(), 'version': float(basicFeaturesetParams['version']), 'created': basicFeatureset['created'], 'creator': { '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser) }, 'globalFeatures': basicFeaturesetParams['globalFeatures'], 'localFeatures': basicFeaturesetParams['localFeatures'] }, resp.json) # Ensure that normal users don't get private creator info resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='GET', user=basicUser) self.assertStatusOk(resp) self.assertDictEqual({ '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser) }, resp.json['creator']) # Ensure that study admin users do get private creator info resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='GET', user=studyAdminUser) self.assertStatusOk(resp) self.assertDictEqual({ '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser), 'firstName': studyAdminUser['firstName'], 'lastName': studyAdminUser['lastName'], 'login': studyAdminUser['login'] }, resp.json['creator']) # Try to delete a featureset as anonymous resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='DELETE') self.assertStatus(resp, 401) # Try to delete a featureset without privileges resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=basicUser) self.assertStatus(resp, 403) # Try to delete a featureset being used by a study resp = self.request( path='/study', method='POST', user=studyAdminUser, type='application/json', body=json.dumps({ 'name': 'Test Study', 'featuresetId': str(basicFeatureset['_id']), 'userIds': [], 'imageIds': [] })) self.assertStatusOk(resp) testStudy = Study.load(resp.json['_id'], force=True) resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=studyAdminUser) self.assertStatus(resp, 409) resp = self.request( path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) # TODO: Use the Study API, once it exists Study.remove(testStudy) # Delete an unused featureset resp = self.request( path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=studyAdminUser, isJson=False) self.assertStatus(resp, 204) resp = self.request( path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(resp.json, [])
def testFeatureset(self): Featureset = self.model('featureset', 'isic_archive') Group = self.model('group') Study = self.model('study', 'isic_archive') User = self.model('user', 'isic_archive') # Create a basic user resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'basic', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) basicUser = User.findOne({'login': '******'}) # Create a study admin user resp = self.request(path='/user', method='POST', params={ 'email': '*****@*****.**', 'login': '******', 'firstName': 'study admin', 'lastName': 'user', 'password': '******' }) self.assertStatusOk(resp) studyAdminUser = User.findOne({'login': '******'}) studyAdminsGroup = Group.findOne({'name': 'Study Administrators'}) Group.addUser(studyAdminsGroup, studyAdminUser, level=AccessType.READ) basicFeaturesetParams = { 'name': ' Basic ', 'version': '1.0', 'globalFeatures': [{ 'id': 'quality', 'name': ['Quality'], 'options': [{ 'id': 'acceptable', 'name': 'Acceptable' }, { 'id': 'unacceptable', 'name': 'Unacceptable' }], 'type': 'radio' }, { 'id': 'diagnosis', 'name': ['Diagnosis'], 'options': [{ 'id': 'benign', 'name': 'Benign' }, { 'id': 'indeterminate', 'name': 'Indeterminate' }, { 'id': 'malignant', 'name': 'Malignant' }], 'type': 'radio' }], 'localFeatures': [{ 'id': 'lesion', 'name': ['Lesion'], 'type': 'superpixel' }, { 'id': 'skin', 'name': ['Normal Skin'], 'type': 'superpixel' }] } # Try to create a featureset as anonymous resp = self.request(path='/featureset', method='POST', params=basicFeaturesetParams) self.assertStatus(resp, 401) # Try to create a featureset without privileges resp = self.request(path='/featureset', method='POST', user=basicUser, params=basicFeaturesetParams) self.assertStatus(resp, 403) # Try to create a featureset with an empty name tempFeaturesetParams = basicFeaturesetParams.copy() tempFeaturesetParams['name'] = '' resp = self.request(path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(tempFeaturesetParams)) self.assertStatus(resp, 400) # Try to create a featureset with an invalid version tempFeaturesetParams = basicFeaturesetParams.copy() tempFeaturesetParams['version'] = 'foo' resp = self.request(path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(tempFeaturesetParams)) self.assertStatus(resp, 400) # TODO: Try to create a featureset with invalid features # List all (nonexistent) featuresets resp = self.request(path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(resp.json, []) # Create a valid featureset resp = self.request(path='/featureset', method='POST', user=studyAdminUser, type='application/json', body=json.dumps(basicFeaturesetParams)) self.assertStatusOk(resp) # Lookup the database entry, for access to non-deterministic details # like '_id' and 'created' basicFeatureset = Featureset.load(resp.json['_id']) # Check the list of all featuresets resp = self.request(path='/featureset', method='GET') self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 1) self.assertDictEqual( { '_id': str(basicFeatureset['_id']), 'name': basicFeaturesetParams['name'].strip(), 'version': float(basicFeaturesetParams['version']), }, resp.json[0]) # Check featureset details as anonymous resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='GET') self.assertStatusOk(resp) # Parse the "created" field first, to make comparison easier self.assertIn('created', resp.json) resp.json['created'] = parseTimestamp(resp.json['created']) self.assertDictEqual( { '_id': str(basicFeatureset['_id']), '_modelType': 'featureset', 'name': basicFeaturesetParams['name'].strip(), 'version': float(basicFeaturesetParams['version']), 'created': basicFeatureset['created'], 'creator': { '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser) }, 'globalFeatures': basicFeaturesetParams['globalFeatures'], 'localFeatures': basicFeaturesetParams['localFeatures'] }, resp.json) # Ensure that normal users don't get private creator info resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='GET', user=basicUser) self.assertStatusOk(resp) self.assertDictEqual( { '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser) }, resp.json['creator']) # Ensure that study admin users do get private creator info resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='GET', user=studyAdminUser) self.assertStatusOk(resp) self.assertDictEqual( { '_id': str(studyAdminUser['_id']), 'name': User.obfuscatedName(studyAdminUser), 'firstName': studyAdminUser['firstName'], 'lastName': studyAdminUser['lastName'], 'login': studyAdminUser['login'] }, resp.json['creator']) # Try to delete a featureset as anonymous resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='DELETE') self.assertStatus(resp, 401) # Try to delete a featureset without privileges resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=basicUser) self.assertStatus(resp, 403) # Try to delete a featureset being used by a study resp = self.request(path='/study', method='POST', user=studyAdminUser, type='application/json', body=json.dumps({ 'name': 'Test Study', 'featuresetId': str(basicFeatureset['_id']), 'userIds': [], 'imageIds': [] })) self.assertStatusOk(resp) testStudy = Study.load(resp.json['_id'], force=True) resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=studyAdminUser) self.assertStatus(resp, 409) resp = self.request(path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) # TODO: Use the Study API, once it exists Study.remove(testStudy) # Delete an unused featureset resp = self.request(path='/featureset/%s' % basicFeatureset['_id'], method='DELETE', user=studyAdminUser, isJson=False) self.assertStatus(resp, 204) resp = self.request(path='/featureset', method='GET') self.assertStatusOk(resp) self.assertEqual(resp.json, [])
def testUploadDataset(self): User = self.model('user', 'isic_archive') # Create users reviewerUser = self._createReviewerUser() uploaderUser = self._createUploaderUser() adminUser = self._createSiteAdminUser() # Create and upload two ZIP files of images publicDataset = self._uploadDataset( uploaderUser=uploaderUser, zipName='test_zip_1', zipContentNames=[ 'test_1_small_1.jpg', 'test_1_small_2.jpg', 'test_1_large_1.jpg' ], datasetName='test_dataset_1', datasetDescription='A public test dataset') privateDataset = self._uploadDataset( uploaderUser=uploaderUser, zipName='test_zip_2', zipContentNames=['test_1_small_3.jpg', 'test_1_large_2.jpg'], datasetName='test_dataset_2', datasetDescription='A private test dataset') # Ensure that ordinary users aren't getting review tasks resp = self.request(path='/task/me/review', method='GET') self.assertStatus(resp, 401) resp = self.request(path='/task/me/review', method='GET', user=uploaderUser) self.assertStatus(resp, 403) # Ensure that reviewer users are getting tasks resp = self.request(path='/task/me/review', method='GET', user=reviewerUser) self.assertStatusOk(resp) reviewTasks = resp.json self.assertEqual(len(reviewTasks), 2) self.assertIn( { 'dataset': { '_id': str(publicDataset['_id']), 'name': publicDataset['name'] }, 'count': 3 }, reviewTasks) self.assertIn( { 'dataset': { '_id': str(privateDataset['_id']), 'name': privateDataset['name'] }, 'count': 2 }, reviewTasks) # Ensure that review task redirects are working resp = self.request(path='/task/me/review/redirect', method='GET', user=reviewerUser) self.assertStatus(resp, 400) for reviewTask in reviewTasks: reviewId = reviewTask['dataset']['_id'] resp = self.request(path='/task/me/review/redirect', method='GET', params={'datasetId': reviewId}, user=reviewerUser, isJson=False) self.assertStatus(resp, 307) self.assertDictContainsSubset( {'Location': '/#tasks/review/%s' % reviewId}, resp.headers) # Accept all images resp = self.request(path='/dataset/%s/review' % publicDataset['_id'], method='GET', user=reviewerUser) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 3) imageIds = [image['_id'] for image in resp.json] resp = self.request(path='/dataset/%s/review' % publicDataset['_id'], method='POST', user=reviewerUser, params={ 'accepted': json.dumps(imageIds), 'flagged': [] }) self.assertStatusOk(resp) # Attempt to register metadata as invalid users csvPath = os.path.join(self.testDataDir, 'test_1_metadata.csv') with open(csvPath, 'rb') as csvStream: resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', body=csvStream.read(), type='text/csv', params={'filename': 'test_1_metadata.csv'}) self.assertStatus(resp, 401) # Attempt to register metadata with invalid parameters with open(csvPath, 'rb') as csvStream: resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', body=csvStream.read(), type='text/csv', user=uploaderUser) self.assertStatus(resp, 400) self.assertIn('"filename" is required', resp.json['message'].lower()) with open(csvPath, 'rb') as csvStream: resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', body=csvStream.read(), type='text/csv', user=uploaderUser, params={'filename': ' '}) self.assertStatus(resp, 400) self.assertIn('filename must be specified', resp.json['message'].lower()) # Attempt to list registered metadata as invalid users resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET') self.assertStatus(resp, 401) resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET', user=uploaderUser) self.assertStatus(resp, 403) # List (empty) registered metadata resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='GET', user=reviewerUser) self.assertStatusOk(resp) self.assertEqual(resp.json, []) # Register metadata with dataset self.assertNoMail() with open(csvPath, 'rb') as csvStream: resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], method='POST', body=csvStream.read(), type='text/csv', isJson=False, user=uploaderUser, params={'filename': 'test_1_metadata.csv'}) self.assertStatusOk(resp) # Reviewer user should receive email self.assertMails(count=1) # List registered metadata resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], user=reviewerUser) self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 1) # Check file field self.assertIn('file', resp.json[0]) self.assertIn('_id', resp.json[0]['file']) self.assertIn('name', resp.json[0]['file']) self.assertEqual('test_1_metadata.csv', resp.json[0]['file']['name']) self.assertIn('user', resp.json[0]) # Check user field self.assertDictEqual( { '_id': str(uploaderUser['_id']), 'name': User.obfuscatedName(uploaderUser) }, resp.json[0]['user']) # Check time field self.assertIn('time', resp.json[0]) self.assertLess(parseTimestamp(resp.json[0]['time']), datetime.datetime.utcnow()) metadataFileId = resp.json[0]['file']['_id'] # Test downloading metadata as invalid users resp = self.request(path='/dataset/%s/metadata/%s/download' % (publicDataset['_id'], metadataFileId), method='GET', isJson=False) self.assertStatus(resp, 401) resp = self.request(path='/dataset/%s/metadata/%s/download' % (publicDataset['_id'], metadataFileId), method='GET', user=uploaderUser, isJson=False) self.assertStatus(resp, 403) # Test downloading metadata resp = self.request(path='/dataset/%s/metadata/%s/download' % (publicDataset['_id'], metadataFileId), method='GET', user=reviewerUser, isJson=False) with open(csvPath, 'rb') as csvStream: self.assertEqual(csvStream.read(), self.getBody(resp)) # Test applying metadata resp = self.request(path='/dataset/%s/metadata/%s/apply' % (publicDataset['_id'], metadataFileId), method='POST', user=uploaderUser, params={'save': False}) self.assertStatus(resp, 403) resp = self.request(path='/dataset/%s/metadata/%s/apply' % (publicDataset['_id'], metadataFileId), method='POST', user=reviewerUser, params={'save': False}) self.assertStatusOk(resp) self.assertIn('errors', resp.json) self.assertIn('warnings', resp.json) self.assertEqual(0, len(resp.json['errors'])) self.assertEqual(resp.json['warnings'], [{ 'description': 'on CSV row 4: no images found that match u\'filename\': u\'test_1_small_3.jpg\'' }, { 'description': 'on CSV row 6: no images found that match u\'filename\': u\'test_1_large_2.jpg\'' }, { 'description': 'unrecognized field u\'age_approx\' will be added to unstructured metadata' }, { 'description': 'unrecognized field u\'isic_source_name\' will be added to unstructured metadata' }]) # Test removing metadata as site admin resp = self.request(path='/dataset/%s/metadata/%s' % (publicDataset['_id'], metadataFileId), method='DELETE', user=adminUser, isJson=False) self.assertStatus(resp, 204) resp = self.request(path='/dataset/%s/metadata' % publicDataset['_id'], user=reviewerUser) self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 0)