def __init__(self): super(Collection, self).__init__() self.resourceName = 'collection' self._model = CollectionModel() self.route('DELETE', (':id',), self.deleteCollection) self.route('GET', (), self.find) self.route('GET', (':id',), self.getCollection) self.route('GET', (':id', 'details'), self.getCollectionDetails) self.route('GET', (':id', 'download'), self.downloadCollection) self.route('GET', (':id', 'access'), self.getCollectionAccess) self.route('POST', (), self.createCollection) self.route('PUT', (':id',), self.updateCollection) self.route('PUT', (':id', 'access'), self.updateCollectionAccess)
class Collection(Resource): """API Endpoint for collections.""" def __init__(self): super(Collection, self).__init__() self.resourceName = 'collection' self._model = CollectionModel() self.route('DELETE', (':id',), self.deleteCollection) self.route('GET', (), self.find) self.route('GET', (':id',), self.getCollection) self.route('GET', (':id', 'details'), self.getCollectionDetails) self.route('GET', (':id', 'download'), self.downloadCollection) self.route('GET', (':id', 'access'), self.getCollectionAccess) self.route('POST', (), self.createCollection) self.route('PUT', (':id',), self.updateCollection) self.route('PUT', (':id', 'access'), self.updateCollectionAccess) @access.public(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('List or search for collections.') .responseClass('Collection', array=True) .param('text', 'Pass this to perform a text search for collections.', required=False) .pagingParams(defaultSort='name') ) def find(self, text, limit, offset, sort): user = self.getCurrentUser() if text is not None: return self._model.textSearch(text, user=user, limit=limit, offset=offset) return self._model.list(user=user, offset=offset, limit=limit, sort=sort) @access.user(scope=TokenScope.DATA_WRITE) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Create a new collection.') .responseClass('Collection') .param('name', 'Name for the collection. Must be unique.') .param('description', 'Collection description.', required=False) .param('public', 'Whether the collection should be publicly visible.', required=False, dataType='boolean', default=False) .errorResponse() .errorResponse('You are not authorized to create collections.', 403) ) def createCollection(self, name, description, public): user = self.getCurrentUser() if not self._model.hasCreatePrivilege(user): raise AccessException('You are not authorized to create collections.') return self._model.createCollection( name=name, description=description, public=public, creator=user) @access.public(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Get a collection by ID.') .responseClass('Collection') .modelParam('id', model=CollectionModel, level=AccessType.READ) .errorResponse('ID was invalid.') .errorResponse('Read permission denied on the collection.', 403) ) def getCollection(self, collection): return collection @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Get detailed information about a collection.') .modelParam('id', model=CollectionModel, level=AccessType.READ) .errorResponse() .errorResponse('Read access was denied on the collection.', 403) ) def getCollectionDetails(self, collection): return { 'nFolders': self._model.countFolders( collection, user=self.getCurrentUser(), level=AccessType.READ) } @access.cookie @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Download an entire collection as a zip archive.') .modelParam('id', model=CollectionModel, level=AccessType.READ) .jsonParam('mimeFilter', 'JSON list of MIME types to include.', requireArray=True, required=False) .produces('application/zip') .errorResponse('ID was invalid.') .errorResponse('Read access was denied for the collection.', 403) ) def downloadCollection(self, collection, mimeFilter): setResponseHeader('Content-Type', 'application/zip') setContentDisposition(collection['name'] + '.zip') def stream(): zip = ziputil.ZipGenerator(collection['name']) for (path, file) in self._model.fileList( collection, user=self.getCurrentUser(), subpath=False, mimeFilter=mimeFilter): for data in zip.addFile(file, path): yield data yield zip.footer() return stream @access.user(scope=TokenScope.DATA_OWN) @autoDescribeRoute( Description('Get the access control list for a collection.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def getCollectionAccess(self, collection): return self._model.getFullAccessList(collection) @access.user(scope=TokenScope.DATA_OWN) @filtermodel(model=CollectionModel, addFields={'access'}) @autoDescribeRoute( Description('Set the access control list for a collection.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .jsonParam('access', 'The access control list as JSON.', requireObject=True) .jsonParam('publicFlags', 'List of public access flags to set on the collection.', required=False, requireArray=True) .param('public', 'Whether the collection should be publicly visible.', dataType='boolean', required=False) .param('recurse', 'Whether the policies should be applied to all ' 'folders under this collection as well.', dataType='boolean', default=False, required=False) .param('progress', 'If recurse is set to True, this controls whether ' 'progress notifications will be sent.', dataType='boolean', default=False, required=False) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def updateCollectionAccess(self, collection, access, public, recurse, progress, publicFlags): user = self.getCurrentUser() progress = progress and recurse with ProgressContext(progress, user=user, title='Updating permissions', message='Calculating progress...') as ctx: if progress: ctx.update(total=self._model.subtreeCount( collection, includeItems=False, user=user, level=AccessType.ADMIN)) return self._model.setAccessList( collection, access, save=True, user=user, recurse=recurse, progress=ctx, setPublic=public, publicFlags=publicFlags) @access.user(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Edit a collection by ID.') .responseClass('Collection') .modelParam('id', model=CollectionModel, level=AccessType.WRITE) .param('name', 'Unique name for the collection.', required=False, strip=True) .param('description', 'Collection description.', required=False, strip=True) .errorResponse('ID was invalid.') .errorResponse('Write permission denied on the collection.', 403) ) def updateCollection(self, collection, name, description): if name is not None: collection['name'] = name if description is not None: collection['description'] = description return self._model.updateCollection(collection) @access.user(scope=TokenScope.DATA_OWN) @autoDescribeRoute( Description('Delete a collection by ID.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def deleteCollection(self, collection): self._model.remove(collection) return {'message': 'Deleted collection %s.' % collection['name']}
def testConsistencyCheck(self): user = self.users[0] c1 = Collection().createCollection('c1', user) f1 = Folder().createFolder(c1, 'f1', parentType='collection') Folder().createFolder(c1, 'f2', parentType='collection') f3 = Folder().createFolder(user, 'f3', parentType='user') Folder().createFolder(user, 'f4', parentType='user') i1 = Item().createItem('i1', user, f1) i2 = Item().createItem('i2', user, f1) Item().createItem('i3', user, f1) i4 = Item().createItem('i4', user, f3) Item().createItem('i5', user, f3) Item().createItem('i6', user, f3) assetstore = {'_id': 0} File().createFile(user, i1, 'foo', 7, assetstore) File().createFile(user, i1, 'foo', 13, assetstore) File().createFile(user, i2, 'foo', 19, assetstore) File().createFile(user, i4, 'foo', 23, assetstore) self.assertEqual(39, Collection().load(c1['_id'], force=True)['size']) self.assertEqual(39, Folder().load(f1['_id'], force=True)['size']) self.assertEqual(23, Folder().load(f3['_id'], force=True)['size']) self.assertEqual(20, Item().load(i1['_id'], force=True)['size']) self.assertEqual(23, User().load(user['_id'], force=True)['size']) resp = self.request(path='/system/check', user=user, method='PUT') self.assertStatusOk(resp) self.assertEqual(resp.json['baseParentsFixed'], 0) self.assertEqual(resp.json['orphansRemoved'], 0) self.assertEqual(resp.json['sizesChanged'], 0) Item().update({'_id': i1['_id']}, update={'$set': { 'baseParentId': None }}) resp = self.request(path='/system/check', user=user, method='PUT') self.assertStatusOk(resp) self.assertEqual(resp.json['baseParentsFixed'], 1) self.assertEqual(resp.json['orphansRemoved'], 0) self.assertEqual(resp.json['sizesChanged'], 0) Collection().update({'_id': c1['_id']}, update={'$set': {'size': 0}}) Folder().update({'_id': f1['_id']}, update={'$set': {'size': 0}}) Item().update({'_id': i1['_id']}, update={'$set': {'size': 0}}) resp = self.request(path='/system/check', user=user, method='PUT') self.assertStatusOk(resp) self.assertEqual(resp.json['baseParentsFixed'], 0) self.assertEqual(resp.json['orphansRemoved'], 0) self.assertEqual(resp.json['sizesChanged'], 3) self.assertEqual(39, Collection().load(c1['_id'], force=True)['size']) self.assertEqual(39, Folder().load(f1['_id'], force=True)['size']) self.assertEqual(23, Folder().load(f3['_id'], force=True)['size']) self.assertEqual(20, Item().load(i1['_id'], force=True)['size']) self.assertEqual(23, User().load(user['_id'], force=True)['size']) Folder().collection.delete_one({'_id': f3['_id']}) resp = self.request(path='/system/check', user=user, method='PUT') self.assertStatusOk(resp) self.assertEqual(resp.json['baseParentsFixed'], 0) self.assertEqual(resp.json['orphansRemoved'], 3) self.assertEqual(resp.json['sizesChanged'], 0) self.assertEqual(0, User().load(user['_id'], force=True)['size'])
def lookUpPath(path, user=None, test=False, filter=True, force=False): """ Look up a resource in the data hierarchy by path. :param path: path of the resource :param user: user with correct privileges to access path :param test: defaults to false, when set to true will return None instead of throwing exception when path doesn't exist :type test: bool :param filter: Whether the returned model should be filtered. :type filter: bool :param force: if True, don't validate the access. :type force: bool """ path = path.lstrip('/') pathArray = split(path) model = pathArray[0] if model == 'user': username = pathArray[1] parent = User().findOne({'login': username}) if parent is None: if test: return {'model': None, 'document': None} else: raise ResourcePathNotFound('User not found: %s' % username) elif model == 'collection': collectionName = pathArray[1] parent = Collection().findOne({'name': collectionName}) if parent is None: if test: return {'model': None, 'document': None} else: raise ResourcePathNotFound('Collection not found: %s' % collectionName) else: raise ValidationException('Invalid path format') try: document = parent if not force: ModelImporter.model(model).requireAccess(document, user) for token in pathArray[2:]: document, model = lookUpToken(token, model, document) if not force: ModelImporter.model(model).requireAccess(document, user) except (ValidationException, AccessException): # We should not distinguish the response between access and validation errors so that # adversarial users cannot discover the existence of data they don't have access to by # looking up a path. if test: return {'model': None, 'document': None} else: raise ResourcePathNotFound('Path not found: %s' % path) if filter: document = ModelImporter.model(model).filter(document, user) return {'model': model, 'document': document}
def loadStudyCollection(self): # assumes collection has been created by provision_utility # TODO: cache this value return Collection().findOne({'name': 'Annotation Studies'})
def testDownload(self): collection = Collection().createCollection('collection1', public=True) folder = Folder().createFolder(collection, 'folder1', parentType='collection', public=True) item = Item().createItem('item1', self.admin, folder) # Path to test files file1Path = os.path.join(self.filesDir, 'txt1.txt') file2Path = os.path.join(self.filesDir, 'txt2.txt') # Upload files to item with open(file1Path, 'rb') as fp: file1 = Upload().uploadFromFile(fp, os.path.getsize(file1Path), 'txt1.txt', parentType='item', parent=item, user=self.admin) with open(file2Path, 'rb') as fp: file2 = Upload().uploadFromFile(fp, os.path.getsize(file2Path), 'txt2.txt', mimeType='image/jpeg', parentType='item', parent=item, user=self.admin) # Download item and its files several times and ensure downloads are recorded # Each file is downloaded 10 times for _ in range(0, 5): self._downloadItem(item['_id']) self._downloadFile(file1['_id']) self._downloadFile(file2['_id']) # Download each file 1 time by downloading parent folder self._downloadFolder(folder['_id']) # Download each file over 2 requests self._downloadFileInTwoChunks(file1['_id']) self._downloadFileInTwoChunks(file2['_id']) # Download each file partially, adding 1 to start and 4 to requested self._downloadPartialFile(file1['_id']) self._downloadPartialFile(file2['_id']) # Download entire collection # Each file is downloaded 1 additional time path = '/collection/%s/download' % collection['_id'] resp = self.request(path, user=self.admin, isJson=False) # Iterate through generator to trigger download events for data in resp.body: data # Download collection filtered by mime type # file2 is downloaded one additional time path = '/collection/%s/download' % collection['_id'] resp = self.request(path, user=self.admin, isJson=False, method='GET', params={ 'id': collection['_id'], 'mimeFilter': json.dumps(['image/jpeg']) }) # iterate through generator to trigger download events for data in resp.body: data self._checkDownloadsCount(file1['_id'], 14, 18, 13) self._checkDownloadsCount(file2['_id'], 15, 19, 14)
def createDataset(self, name, description, license, attribution, owner, creatorUser): now = datetime.datetime.utcnow() # Validate before saving anything dataset = self.save({ # Public informational data 'name': name, 'description': description, 'license': license, 'attribution': attribution, # Public Girder data 'created': now, 'updated': now, # Private informational data 'owner': owner, 'metadataFiles': [], # Private Girder data 'creatorId': creatorUser['_id'], 'folderId': None, 'public': False, 'access': { 'users': [], 'groups': [] } }) # Create folder and add it to the dataset datasetFolder = Folder().createFolder( parent=Collection().findOne({'name': 'Lesion Images'}), name=dataset['name'], parentType='collection', creator=creatorUser, public=dataset['public'], allowRename=False, reuseExisting=False) dataset['folderId'] = datasetFolder['_id'] self.update( {'_id': dataset['_id']}, {'$set': {'folderId': dataset['folderId']}} ) # Set default accesses (overwriting inherited accesses on the folder) dataset = self.setAccessList( doc=dataset, access={ 'users': [ # Allow the creator to write { 'id': creatorUser['_id'], 'level': AccessType.WRITE, 'flags': [] } ], 'groups': [ # Allow reviewers to admin (so they can delete the dataset) { 'id': Group().findOne({'name': 'Dataset QC Reviewers'})['_id'], 'level': AccessType.ADMIN, 'flags': [] } ] } ) return dataset
class Collection(Resource): """API Endpoint for collections.""" def __init__(self): super(Collection, self).__init__() self.resourceName = 'collection' self._model = CollectionModel() self.route('DELETE', (':id',), self.deleteCollection) self.route('GET', (), self.find) self.route('GET', (':id',), self.getCollection) self.route('GET', (':id', 'details'), self.getCollectionDetails) self.route('GET', (':id', 'download'), self.downloadCollection) self.route('GET', (':id', 'access'), self.getCollectionAccess) self.route('POST', (), self.createCollection) self.route('PUT', (':id',), self.updateCollection) self.route('PUT', (':id', 'access'), self.updateCollectionAccess) @access.public(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('List or search for collections.') .responseClass('Collection', array=True) .param('text', 'Pass this to perform a text search for collections.', required=False) .pagingParams(defaultSort='name') ) def find(self, text, limit, offset, sort): user = self.getCurrentUser() if text is not None: return self._model.textSearch(text, user=user, limit=limit, offset=offset) return self._model.list(user=user, offset=offset, limit=limit, sort=sort) @access.user(scope=TokenScope.DATA_WRITE) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Create a new collection.') .responseClass('Collection') .param('name', 'Name for the collection. Must be unique.') .param('description', 'Collection description.', required=False) .param('public', 'Whether the collection should be publicly visible.', required=False, dataType='boolean', default=False) .errorResponse() .errorResponse('You are not authorized to create collections.', 403) ) def createCollection(self, name, description, public): user = self.getCurrentUser() if not self._model.hasCreatePrivilege(user): raise AccessException('You are not authorized to create collections.') return self._model.createCollection( name=name, description=description, public=public, creator=user) @access.public(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Get a collection by ID.') .responseClass('Collection') .modelParam('id', model=CollectionModel, level=AccessType.READ) .errorResponse('ID was invalid.') .errorResponse('Read permission denied on the collection.', 403) ) def getCollection(self, collection): return collection @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( Description('Get detailed information about a collection.') .modelParam('id', model=CollectionModel, level=AccessType.READ) .errorResponse() .errorResponse('Read access was denied on the collection.', 403) ) def getCollectionDetails(self, collection): return { 'nFolders': self._model.countFolders( collection, user=self.getCurrentUser(), level=AccessType.READ) } @access.public(scope=TokenScope.DATA_READ, cookie=True) @autoDescribeRoute( Description('Download an entire collection as a zip archive.') .modelParam('id', model=CollectionModel, level=AccessType.READ) .jsonParam('mimeFilter', 'JSON list of MIME types to include.', requireArray=True, required=False) .produces('application/zip') .errorResponse('ID was invalid.') .errorResponse('Read access was denied for the collection.', 403) ) def downloadCollection(self, collection, mimeFilter): setResponseHeader('Content-Type', 'application/zip') setContentDisposition(collection['name'] + '.zip') def stream(): zip = ziputil.ZipGenerator(collection['name']) for (path, file) in self._model.fileList( collection, user=self.getCurrentUser(), subpath=False, mimeFilter=mimeFilter): for data in zip.addFile(file, path): yield data yield zip.footer() return stream @access.user(scope=TokenScope.DATA_OWN) @autoDescribeRoute( Description('Get the access control list for a collection.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def getCollectionAccess(self, collection): return self._model.getFullAccessList(collection) @access.user(scope=TokenScope.DATA_OWN) @filtermodel(model=CollectionModel, addFields={'access'}) @autoDescribeRoute( Description('Set the access control list for a collection.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .jsonParam('access', 'The access control list as JSON.', requireObject=True) .jsonParam('publicFlags', 'List of public access flags to set on the collection.', required=False, requireArray=True) .param('public', 'Whether the collection should be publicly visible.', dataType='boolean', required=False) .param('recurse', 'Whether the policies should be applied to all ' 'folders under this collection as well.', dataType='boolean', default=False, required=False) .param('progress', 'If recurse is set to True, this controls whether ' 'progress notifications will be sent.', dataType='boolean', default=False, required=False) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def updateCollectionAccess(self, collection, access, public, recurse, progress, publicFlags): user = self.getCurrentUser() progress = progress and recurse with ProgressContext(progress, user=user, title='Updating permissions', message='Calculating progress...') as ctx: if progress: ctx.update(total=self._model.subtreeCount( collection, includeItems=False, user=user, level=AccessType.ADMIN)) return self._model.setAccessList( collection, access, save=True, user=user, recurse=recurse, progress=ctx, setPublic=public, publicFlags=publicFlags) @access.user(scope=TokenScope.DATA_READ) @filtermodel(model=CollectionModel) @autoDescribeRoute( Description('Edit a collection by ID.') .responseClass('Collection') .modelParam('id', model=CollectionModel, level=AccessType.WRITE) .param('name', 'Unique name for the collection.', required=False, strip=True) .param('description', 'Collection description.', required=False, strip=True) .errorResponse('ID was invalid.') .errorResponse('Write permission denied on the collection.', 403) ) def updateCollection(self, collection, name, description): if name is not None: collection['name'] = name if description is not None: collection['description'] = description return self._model.updateCollection(collection) @access.user(scope=TokenScope.DATA_OWN) @autoDescribeRoute( Description('Delete a collection by ID.') .modelParam('id', model=CollectionModel, level=AccessType.ADMIN) .errorResponse('ID was invalid.') .errorResponse('Admin permission denied on the collection.', 403) ) def deleteCollection(self, collection): self._model.remove(collection) return {'message': 'Deleted collection %s.' % collection['name']}
def getTCGACollection(self): """Get the unique TCGA collection from the settings collection.""" tcga = Setting().get(TCGACollectionSettingKey) if tcga is None: raise Exception('TCGA collection id not initialized in settings') return Collection().load(tcga, force=True)
def _validateStudiesColl(doc): Collection().load(doc['value'], exc=True, force=True)
def setUp(self): super(ResourceOperationsTestCase, self).setUp() users = ( { "email": "*****@*****.**", "login": "******", "firstName": "Jane", "lastName": "Austin", "password": "******", }, { "email": "*****@*****.**", "login": "******", "firstName": "Sally", "lastName": "User", "password": "******", }, ) self.users = { user["login"]: User().createUser(**user) for user in users } self.public_root = tempfile.mkdtemp() self.private_root = tempfile.mkdtemp() self.base_collection = Collection().createCollection( "Virtual Resources", creator=self.users["admin"], public=True, reuseExisting=True, ) self.public_folder = Folder().createFolder( self.base_collection, "public", parentType="collection", public=True, reuseExisting=True, ) self.public_folder.update(dict(fsPath=self.public_root, isMapping=True)) self.public_folder = Folder().save(self.public_folder) self.regular_folder = Folder().createFolder( self.base_collection, "public_no_map", parentType="collection", public=True, reuseExisting=True, ) self.private_folder = Folder().createFolder( self.base_collection, "private", creator=self.users["sally"], parentType="collection", public=False, reuseExisting=True, ) self.private_folder.update( dict(fsPath=self.private_root, isMapping=True)) self.private_folder = Folder().save(self.private_folder)
def setUp(self): super(SlicerExtensionManagerTest, self).setUp() self.maxDiff = None self._MAX_CHUNK_SIZE = 1024 * 1024 * 64 self._extensionNameTemplate = '{app_revision}_{os}_{arch}_{baseName}_{revision}' self.dataDir = os.path.join(os.environ['GIRDER_TEST_DATA_PREFIX'], 'plugins', 'slicer_extension_manager') self._user = User().createUser('usr0', 'passwd', 'tst', 'usr', '*****@*****.**') self._collection = Collection().createCollection( 'testCollection', creator=self._user, description='Contain applications') self._app = Folder().createFolder(parent=self._collection, name='application', description='app description', parentType='Collection', public=True, creator=self._user) self._app = Folder().setMetadata( self._app, {'extensionNameTemplate': self._extensionNameTemplate}) self._nightly = Folder().createFolder( parent=self._app, name=constants.NIGHTLY_RELEASE_NAME, description='Uploaded each night, always up-to-date', parentType='Folder', public=True, creator=self._user) self._release = Folder().createFolder( parent=self._app, name='release', description='release description', parentType='Folder', public=True, creator=self._user) self._release = Folder().setMetadata(self._release, {'revision': '12345'}) self._extensions = { 'extension1': { 'name': '12345_linux_i386_slicerExt_35333', 'meta': { 'os': 'linux', 'arch': 'i386', 'baseName': 'slicerExt', 'repository_type': 'git', 'repository_url': 'http://slicer.com/extension/Ext', 'revision': '35333', 'app_revision': '12345', 'packagetype': 'installer', 'codebase': 'SL4', 'description': 'Extension for Slicer 4' } }, 'extension2': { 'name': '112233_win_i386_slicerExt1_54342', 'meta': { 'os': 'win', 'arch': 'i386', 'baseName': 'slicerExt1', 'repository_type': 'git', 'repository_url': 'http://slicer.com/extension/Ext', 'revision': '54342', 'app_revision': '112233', 'packagetype': 'installer', 'codebase': 'SL4', 'description': 'Extension for Slicer 4 new version' } }, 'extension3': { 'name': '112233_linux_amd64_slicerExt2_542', 'meta': { 'os': 'linux', 'arch': 'amd64', 'baseName': 'slicerExt2', 'repository_type': 'gitlab', 'repository_url': 'http://slicer.com/extension/Ext', 'revision': '542', 'app_revision': '112233', 'packagetype': 'zip', 'codebase': 'SL434334', 'description': 'Extension for Slicer 4 new version' } } }
def testMongoSearch(self): """ Test resource/mongo_search endpoint """ # Create a bunch of searchable documents admin = { 'email': '*****@*****.**', 'login': '******', 'firstName': 'Admin', 'lastName': 'Last', 'password': '******', 'admin': True } admin = User().createUser(**admin) user = { 'email': '*****@*****.**', 'login': '******', 'firstName': 'First', 'lastName': 'Last', 'password': '******', 'admin': False } user = User().createUser(**user) coll1 = { 'name': 'Test Collection', 'description': 'magic words. And more magic.', 'public': True, 'creator': admin } coll1 = Collection().createCollection(**coll1) coll2 = { 'name': 'Magic collection', 'description': 'private', 'public': False, 'creator': admin } coll2 = Collection().createCollection(**coll2) Collection().setUserAccess(coll2, user, level=AccessType.READ, save=True) folder1 = { 'parent': coll1, 'parentType': 'collection', 'name': 'Public test folder' } folder1 = Folder().createFolder(**folder1) Folder().setUserAccess(folder1, user, level=AccessType.READ, save=False) Folder().setPublic(folder1, True, save=True) folder2 = { 'parent': coll2, 'parentType': 'collection', 'name': 'Private test folder' } folder2 = Folder().createFolder(**folder2) Folder().setUserAccess(folder2, user, level=AccessType.NONE, save=True) item1 = {'name': 'Public object', 'creator': admin, 'folder': folder1} item1 = Item().createItem(**item1) item2 = {'name': 'Secret object', 'creator': admin, 'folder': folder2} item2 = Item().createItem(**item2) # Grab the default user folders resp = self.request(path='/folder', method='GET', user=user, params={ 'parentType': 'user', 'parentId': user['_id'], 'sort': 'name', 'sortdir': 1 }) # First test all of the required parameters. self.ensureRequiredParams(path='/resource/search', required=['q', 'types']) # Now test parameter validation resp = self.request(path='/resource/mongo_search', params={ 'q': 'query', 'type': 'wrong type' }) self.assertStatus(resp, 400) self.assertEqual('Invalid resource type: wrong type', resp.json['message']) # Test validation of JSON input resp = self.request(path='/resource/mongo_search', params={ 'q': 'not_json', 'type': 'folder' }) self.assertStatus(resp, 400) self.assertEqual(resp.json['message'], 'The query parameter must be a JSON object.') # Ensure searching respects permissions resp = self.request(path='/resource/mongo_search', params={ 'q': bson.json_util.dumps({'name': 'Private'}), 'type': 'folder' }) self.assertStatusOk(resp) self.assertEqual(resp.json, []) resp = self.request(path='/resource/mongo_search', params={ 'q': bson.json_util.dumps({'name': 'Private'}), 'type': 'folder' }, user=user) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertHasKeys(resp.json[0], ('_id', 'name', 'description')) self.assertEqual(len(resp.json[0]), 3) # Test item search resp = self.request(path='/resource/mongo_search', params={ 'q': bson.json_util.dumps( {'folderId': folder1['_id']}), 'type': 'item' }) self.assertStatusOk(resp) self.assertEqual(resp.json, [{ '_id': str(item1['_id']), 'name': 'Public object', 'description': '', 'folderId': str(folder1['_id']) }]) resp = self.request(path='/resource/mongo_search', params={ 'q': bson.json_util.dumps( {'folderId': folder2['_id']}), 'type': 'item' }) self.assertStatusOk(resp) self.assertEqual(resp.json, []) resp = self.request(path='/resource/mongo_search', params={ 'q': bson.json_util.dumps( {'folderId': folder2['_id']}), 'type': 'item' }, user=admin) self.assertStatusOk(resp) self.assertEqual(resp.json, [{ '_id': str(item2['_id']), 'name': 'Secret object', 'description': '', 'folderId': str(folder2['_id']) }])
def testResourceItems(self): # Create some resources to use in the tests self.collection = Collection().createCollection( 'collection A', self.admin) self.colFolderA = Folder().createFolder(self.collection, 'folder A', parentType='collection', creator=self.admin) self.colFolderB = Folder().createFolder(self.collection, 'folder B', parentType='collection', creator=self.admin) self.colFolderC = Folder().createFolder(self.colFolderA, 'folder C', creator=self.admin) self.colItemA1 = Item().createItem('item A1', self.admin, self.colFolderA) self.colItemB1 = Item().createItem('item B1', self.admin, self.colFolderB) self.colItemB2 = Item().createItem('item B2', self.admin, self.colFolderB) self.colItemC1 = Item().createItem('item C1', self.admin, self.colFolderC) self.colItemC2 = Item().createItem('item C2', self.admin, self.colFolderC) self.colItemC3 = Item().createItem('item C3', self.admin, self.colFolderC) self.itemPub1 = Item().createItem('item Public 1', self.admin, self.publicFolder) self.itemPriv1 = Item().createItem('item Private 1', self.admin, self.privateFolder) self.folderD = Folder().createFolder(self.publicFolder, 'folder D', creator=self.admin) self.itemD1 = Item().createItem('item D1', self.admin, self.folderD) self.itemD2 = Item().createItem('item D2', self.admin, self.folderD) # Now test that we get the items we expect # From a user resp = self.request(path='/resource/%s/items' % self.admin['_id'], user=self.admin, params={'type': 'user'}) self.assertStatusOk(resp) items = resp.json self.assertEqual( [item['name'] for item in items], ['item Public 1', 'item D1', 'item D2', 'item Private 1']) # From a collection resp = self.request(path='/resource/%s/items' % self.collection['_id'], user=self.admin, params={'type': 'collection'}) self.assertStatusOk(resp) items = resp.json self.assertEqual( [item['name'] for item in items], ['item A1', 'item C1', 'item C2', 'item C3', 'item B1', 'item B2']) # With sort, limit, and offset resp = self.request(path='/resource/%s/items' % self.collection['_id'], user=self.admin, params={ 'type': 'collection', 'limit': 4, 'offset': 1, 'sort': 'name', 'sortdir': -1 }) self.assertStatusOk(resp) items = resp.json self.assertEqual([item['name'] for item in items], ['item B1', 'item A1', 'item C3', 'item C2']) resp = self.request(path='/resource/%s/items' % self.collection['_id'], user=self.admin, params={ 'type': 'collection', 'limit': 1, 'offset': 0, 'sort': 'name', 'sortdir': -1 }) self.assertStatusOk(resp) items = resp.json self.assertEqual([item['name'] for item in items], ['item B2']) # From a folder resp = self.request(path='/resource/%s/items' % self.colFolderA['_id'], user=self.admin, params={'type': 'folder'}) self.assertStatusOk(resp) items = resp.json self.assertEqual([item['name'] for item in items], ['item A1', 'item C1', 'item C2', 'item C3']) # From a lower folder resp = self.request(path='/resource/%s/items' % self.colFolderC['_id'], user=self.admin, params={'type': 'folder'}) self.assertStatusOk(resp) items = resp.json self.assertEqual([item['name'] for item in items], ['item C1', 'item C2', 'item C3']) # With a bad parameter resp = self.request(path='/resource/%s/items' % self.colFolderC['_id'], user=self.admin, params={'type': 'collection'}) self.assertStatus(resp, 400) self.assertIn('Resource not found', resp.json['message'])
def testResourceMetadata(self): # Create some resources to use in the tests self.collection = Collection().createCollection( 'collection A', self.admin) self.colFolderA = Folder().createFolder(self.collection, 'folder A', parentType='collection', creator=self.admin) self.colItemA1 = Item().createItem('item A1', self.admin, self.colFolderA) self.colItemB1 = Item().createItem('item B1', self.admin, self.colFolderA) resp = self.request(method='PUT', path='/resource/metadata', params={ 'resources': json.dumps( {'item': [str(self.colItemA1['_id'])]}), 'metadata': json.dumps({ 'keya': 'valuea', 'keyb.keyc': 'valuec' }) }) self.assertStatus(resp, 401) resp = self.request(method='PUT', path='/resource/metadata', user=self.admin, params={ 'resources': json.dumps( {'item': [str(self.colItemA1['_id'])]}), 'metadata': json.dumps({ 'keya': 'valuea', 'keyb.keyc': 'valuec' }) }) self.assertStatusOk(resp) self.assertEqual(resp.json, 1) meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta'] self.assertEqual(meta['keya'], 'valuea') self.assertEqual(meta['keyb']['keyc'], 'valuec') resp = self.request( method='PUT', path='/resource/metadata', user=self.admin, params={ 'resources': json.dumps({ 'item': [str(self.colItemA1['_id']), str(self.colItemB1['_id'])], 'folder': [str(self.colFolderA['_id'])] }), 'metadata': json.dumps({ 'keya': 'valuea', 'keyb.keyc': None, 'keyb.keyd': 'valued', }) }) self.assertStatusOk(resp) self.assertEqual(resp.json, 3) meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta'] self.assertEqual(meta['keya'], 'valuea') self.assertNotIn('keyc', meta['keyb']) self.assertEqual(meta['keyb']['keyd'], 'valued') resp = self.request(method='PUT', path='/resource/metadata', user=self.admin, params={ 'resources': json.dumps({ 'item': [ str(self.colItemA1['_id']), ], 'folder': [str(self.colFolderA['_id'])] }), 'metadata': json.dumps({ 'keya': 'valuea', 'keyb.keyc': None, 'keyb.keyd': 'valued', }), 'allowNull': True }) self.assertStatusOk(resp) self.assertEqual(resp.json, 2) meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta'] self.assertEqual(meta['keya'], 'valuea') self.assertIsNone(meta['keyb']['keyc']) self.assertEqual(meta['keyb']['keyd'], 'valued')
def initApp(self, name, app_description, collection_id, collection_name, collection_description, public): """ Create the directory for start a new application. By default, without specifying a ``collection_id``, it will create a new collection name either ``collection_name`` if provided, or **'Applications'**. If the collection 'Applications already exist it will get it. Return the new application (as a folder) that always contain a default sub-folder named 'nightly'. :param name: Name of the new application :param app_description: Description of the new application :param collection_id: Id of the collection within create the application :param collection_name: Name of the collection which will be created :param collection_description: Description of the new collection :param public: Whether the new collection should be publicly visible :return: The new application folder """ creator = self.getCurrentUser() # Load or create the collection that contain the application if collection_id: collection = Collection().load(collection_id, force=True) elif collection_name: collection = list(Collection().find({'name': collection_name}, user=creator)) if not collection: collection = Collection().createCollection( name=collection_name, description=collection_description, public=public, creator=creator) else: collection = collection[0] else: collection = list(Collection().find({'name': 'Applications'}, user=creator)) if not collection: collection = Collection().createCollection( name='Applications', description=collection_description, public=public, creator=creator) else: collection = collection[0] # Create the application if not app_description: app_description = '' app = self._model.createFolder(parent=collection, name=name, description=app_description, parentType='Collection', public=public, creator=creator) # Create the 'nightly' release which will be the default folder when uploading an extension self._model.createFolder( parent=app, name=constants.NIGHTLY_RELEASE_NAME, description='Uploaded each night, always up-to-date', parentType='Folder', public=public, creator=creator) # Set a default template name for extensions in the application, # this can be changed in anytime. return self._model.setMetadata( app, { 'extensionNameTemplate': '{app_revision}_{os}_{arch}_{baseName}_{revision}' })
def reviewImages(self, dataset, acceptedImages, flaggedImages, user): # Avoid circular import from .image import Image # Verify that all images are pending review prereviewFolder = self.prereviewFolder(dataset) if not prereviewFolder: raise GirderException('No Pre-review folder for this dataset.') for image in itertools.chain(acceptedImages, flaggedImages): if image['folderId'] != prereviewFolder['_id']: raise ValidationException('Image %s is not in Pre-review.' % image['_id']) now = datetime.datetime.utcnow() for image in acceptedImages: image = Image().setMetadata(image, { 'reviewed': { 'userId': user['_id'], 'time': now, 'accepted': True } }) Image().move(image, dataset) if flaggedImages: flaggedCollection = Collection().findOne( {'name': 'Flagged Images'}) flaggedFolder = Folder().findOne({ 'name': dataset['name'], 'parentId': flaggedCollection['_id'], 'parentCollection': 'collection' }) if not flaggedFolder: flaggedFolder = Folder().createFolder(parent=flaggedCollection, name=dataset['name'], parentType='collection', public=None, creator=user, allowRename=False, reuseExisting=False) flaggedFolder = Folder().copyAccessPolicies(prereviewFolder, flaggedFolder, save=True) for image in flaggedImages: image = Image().setMetadata( image, { 'reviewed': { 'userId': user['_id'], 'time': now, 'accepted': False } }) Image().move(image, flaggedFolder) # Remove an empty Pre-review folder if (Folder().countItems(prereviewFolder) + Folder().countFolders(prereviewFolder)) == 0: Folder().remove(prereviewFolder)
def testSftpService(self): users = ({ 'email': '*****@*****.**', 'login': '******', 'firstName': 'First', 'lastName': 'Last', 'password': '******' }, { 'email': '*****@*****.**', 'login': '******', 'firstName': 'First', 'lastName': 'Last', 'password': '******' }) admin, user = [User().createUser(**user) for user in users] collections = ({ 'name': 'public collection', 'public': True, 'creator': admin }, { 'name': 'private collection', 'public': False, 'creator': admin }) privateFolder = Folder().findOne({ 'parentCollection': 'user', 'parentId': user['_id'], 'name': 'Private' }) self.assertIsNotNone(privateFolder) Upload().uploadFromFile(six.BytesIO(b'hello world'), size=11, name='test.txt', parentType='folder', parent=privateFolder, user=user) for coll in collections: Collection().createCollection(**coll) client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Incorrect password should raise authentication error with self.assertRaises(paramiko.AuthenticationException): client.connect('localhost', TEST_PORT, username='******', password='******', look_for_keys=False, allow_agent=False) # Authenticate as admin client.connect('localhost', TEST_PORT, username='******', password='******', look_for_keys=False, allow_agent=False) sftpClient = client.open_sftp() self.assertEqual(sftpClient.listdir('/'), ['collection', 'user']) # Listing an invalid top level entity should fail with self.assertRaises(IOError): sftpClient.listdir('/foo') # Test listing of users, collections, and subfolders self.assertEqual(set(sftpClient.listdir('/user/')), {'admin', 'regularuser'}) self.assertEqual(set(sftpClient.listdir('/user/admin')), {'Public', 'Private'}) self.assertEqual(set(sftpClient.listdir('/collection')), {'public collection', 'private collection'}) self.assertEqual(sftpClient.listdir('/user/regularuser/Private'), ['test.txt']) self.assertEqual( sftpClient.listdir('/user/regularuser/Private/test.txt'), ['test.txt']) with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.listdir('/user/nonexistent') with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.file('/user/regularuser/Private') # Read a file using small enough buf size to require multiple chunks internally. file = sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r', bufsize=4) self.assertEqual(file.read(2), b'he') self.assertEqual(file.read(), b'llo world') # Make sure we enforce max buffer length tmp, sftp.MAX_BUF_LEN = sftp.MAX_BUF_LEN, 2 file = sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r', bufsize=4) with self.assertRaises(IOError): file.read() sftp.MAX_BUF_LEN = tmp # Test stat capability info = sftpClient.stat('/user/regularuser/Private') self.assertTrue(stat.S_ISDIR(info.st_mode)) self.assertFalse(stat.S_ISREG(info.st_mode)) self.assertEqual(info.st_mode & 0o777, 0o777) # lstat should also work info = sftpClient.lstat('/user/regularuser/Private/test.txt/test.txt') self.assertFalse(stat.S_ISDIR(info.st_mode)) self.assertTrue(stat.S_ISREG(info.st_mode)) self.assertEqual(info.st_size, 11) self.assertEqual(info.st_mode & 0o777, 0o777) # File stat implementations should agree info = file.stat() self.assertFalse(stat.S_ISDIR(info.st_mode)) self.assertTrue(stat.S_ISREG(info.st_mode)) self.assertEqual(info.st_size, 11) self.assertEqual(info.st_mode & 0o777, 0o777) # Make sure we can stat the top-level entities for path in ('/', '/user', '/collection'): info = sftpClient.stat(path) self.assertTrue(stat.S_ISDIR(info.st_mode)) self.assertFalse(stat.S_ISREG(info.st_mode)) self.assertEqual(info.st_mode & 0o777, 0o777) sftpClient.close() client.close() # Test that any username other than anonymous will fail using auth_none. sock = socket.socket() sock.connect(('localhost', TEST_PORT)) trans = paramiko.Transport(sock) trans.connect() with self.assertRaises(paramiko.ssh_exception.BadAuthenticationType): trans.auth_none('') trans.close() sock.close() sock = socket.socket() sock.connect(('localhost', TEST_PORT)) trans = paramiko.Transport(sock) trans.connect() with self.assertRaises(paramiko.ssh_exception.BadAuthenticationType): trans.auth_none('eponymous') trans.close() sock.close() # Test that a connection can be opened for anonymous access using auth_none. sock = socket.socket() sock.connect(('localhost', TEST_PORT)) trans = paramiko.Transport(sock) trans.connect() trans.auth_none(username='******') sftpClient = paramiko.SFTPClient.from_transport(trans) # Only public data should be visible self.assertEqual(set(sftpClient.listdir('/user')), {'admin', 'regularuser'}) self.assertEqual(sftpClient.listdir('/collection'), ['public collection']) self.assertEqual(sftpClient.listdir('/user/admin'), ['Public']) # Make sure the client cannot distinguish between a resource that does not exist # vs. one they simply don't have read access to. with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.listdir('/user/regularuser/Private') with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r') sftpClient.close() trans.close() sock.close() # Test anonymous access client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect('localhost', TEST_PORT, username='******', password='', look_for_keys=False, allow_agent=False) sftpClient = client.open_sftp() # Only public data should be visible self.assertEqual(set(sftpClient.listdir('/user')), {'admin', 'regularuser'}) self.assertEqual(sftpClient.listdir('/collection'), ['public collection']) self.assertEqual(sftpClient.listdir('/user/admin'), ['Public']) # Make sure the client cannot distinguish between a resource that does not exist # vs. one they simply don't have read access to. with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.listdir('/user/regularuser/Private') with six.assertRaisesRegex(self, IOError, 'No such file'): sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r') sftpClient.close() client.close()
def stats(self, params): drafts = get_or_create_drafts_collection() draft_count = Collection().countFolders(drafts) # TODO no way to publish published_count = 0 user_count = User().collection.count_documents({}) # These aggregate queries are not indexable. # Here are some benchmarks I generated for this endpoint as is: # 0 dandisets | 0.008207440376281738 seconds # 1000 dandisets | 0.019745850563049318 seconds # 2000 dandisets | 0.030080437660217285 seconds # 1000 more dandisets takes about .011 more seconds # Here is the same benchmark but with these aggregate queries removed: # 0 dandisets | 0.004081225395202637 # 1000 dandisets | 0.008220505714416505 # 2000 dandisets | 0.011307811737060547 # 1000 more dandisets takes about 0.0035 more seconds species_count = list(Folder().collection.aggregate([ { "$unwind": "$meta.dandiset.organism" }, { "$group": { "_id": "$meta.dandiset.organism.species" } }, { "$group": { "_id": "0", "count": { "$sum": 1 } } }, ])) if species_count: species_count = species_count[0]["count"] else: species_count = 0 subject_count = list(Folder().collection.aggregate([{ "$group": { "_id": "0", "count": { "$sum": "$meta.dandiset.number_of_subjects" } } }])) if subject_count: subject_count = subject_count[0]["count"] else: subject_count = 0 cell_count = list(Folder().collection.aggregate([{ "$group": { "_id": "0", "count": { "$sum": "$meta.dandiset.number_of_cells" } } }])) if cell_count: cell_count = cell_count[0]["count"] else: cell_count = 0 return { "draft_count": draft_count, "published_count": published_count, "user_count": user_count, "species_count": species_count, "subject_count": subject_count, "cell_count": cell_count, "size": drafts["size"], }
# If there is are no users, create an admin user if User().findOne() is None: User().createUser('admin', 'password', 'Admin', 'Admin', '*****@*****.**') adminUser = User().findOne({'admin': True}) # Configure asset stores if Assetstore().findOne() is None: Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore') Assetstore().createFilesystemAssetstore('SMM', '/Datashare/SMM') Assetstore().createFilesystemAssetstore('GlusterFS', '/glusterfs/SMM') # If we don't have a default task folder, make a task collection and folder if not Setting().get('slicer_cli_web.task_folder'): # Make sure we have a Tasks collection with a Slicer CLI Web Tasks folder if Collection().findOne({'name': 'Tasks'}) is None: Collection().createCollection('Tasks', adminUser) tasksCollection = Collection().findOne({'name': 'Tasks'}) taskFolderName = 'Slicer CLI Web Tasks' if Folder().findOne({ 'name': taskFolderName, 'parentId': tasksCollection['_id'] }) is None: Folder().createFolder(tasksCollection, taskFolderName, parentType='collection', public=True, creator=adminUser) taskFolder = Folder().findOne({ 'name': taskFolderName, 'parentId': tasksCollection['_id']
def get_or_create_drafts_collection(): return Collection().createCollection(DANDI_DRAFTS_COLLECTION_NAME, reuseExisting=True)
def testYAMLConfigFile(server, admin, user, fsAssetstore): # Create some resources to use in the tests collection = Collection().createCollection('collection A', admin) colFolderA = Folder().createFolder(collection, 'folder A', parentType='collection', creator=admin) colFolderB = Folder().createFolder(colFolderA, 'folder B', creator=admin) groupA = Group().createGroup('Group A', admin) resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), method='GET') assert utilities.respStatus(resp) == 200 assert resp.json is None colFolderConfig = Folder().createFolder(collection, '.config', parentType='collection', creator=admin) utilities.uploadText(json.dumps({'keyA': 'value1'}), admin, fsAssetstore, colFolderConfig, 'sample.json') resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id'])) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value1' utilities.uploadText(json.dumps({'keyA': 'value2'}), admin, fsAssetstore, colFolderA, 'sample.json') resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id'])) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value2' utilities.uploadText( json.dumps({ 'keyA': 'value3', 'groups': { 'Group A': { 'access': { 'user': { 'keyA': 'value4' } } } }, 'access': { 'user': { 'keyA': 'value5' }, 'admin': { 'keyA': 'value6' } } }), admin, fsAssetstore, colFolderB, 'sample.json') resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id'])) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value3' resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=user) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value5' resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=admin) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value6' Group().addUser(groupA, user) resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=user) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value4' resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=admin) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value6'
def _testDownloadCollection(self): """ Test downloading an entire collection as a zip file. """ # Create a collection resp = self.request( path='/collection', method='POST', user=self.user, params={ 'name': 'Test Collection' }) self.assertStatusOk(resp) collection = resp.json # Create a folder in the collection resp = self.request( path='/folder', method='POST', user=self.user, params={ 'name': 'Test Folder', 'parentId': collection['_id'], 'parentType': 'collection' }) self.assertStatusOk(resp) test = resp.json contents = os.urandom(64) # Generate random file contents # Upload the file into that subfolder resp = self.request( path='/file', method='POST', user=self.user, params={ 'parentType': 'folder', 'parentId': test['_id'], 'name': 'random.bin', 'size': len(contents) }) self.assertStatusOk(resp) uploadId = resp.json['_id'] # Send the file contents resp = self.request( path='/file/chunk', method='POST', user=self.user, body=contents, params={ 'uploadId': uploadId }, type='application/octet-stream') self.assertStatusOk(resp) # Download the collection path = '/collection/%s/download' % collection['_id'] resp = self.request( path=path, method='GET', user=self.user, isJson=False) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Disposition'], 'attachment; filename="Test Collection.zip"') self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) extracted = zip.read('Test Collection/Test Folder/random.bin') self.assertEqual(extracted, contents) # Make collection public collection = Collection().load(collection['_id'], force=True) collection['public'] = True collection = Collection().save(collection) # Download the collection as anonymous path = '/collection/%s/download' % str(collection['_id']) resp = self.request( path=path, method='GET', user=None, isJson=False) self.assertStatusOk(resp) zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') # Zip file should have no entries self.assertFalse(zip.namelist()) # Upload a known MIME-type file into the collection contents = b'not a jpeg' resp = self.request( path='/file', method='POST', user=self.user, params={ 'parentType': 'folder', 'parentId': test['_id'], 'name': 'fake.jpeg', 'size': len(contents), 'mimeType': 'image/jpeg' }) self.assertStatusOk(resp) uploadId = resp.json['_id'] # Send the file contents resp = self.request( path='/file/chunk', method='POST', user=self.user, body=contents, params={ 'uploadId': uploadId }, type='application/octet-stream') self.assertStatusOk(resp) # Download the collection using a MIME type filter path = '/collection/%s/download' % str(collection['_id']) resp = self.request( path=path, method='GET', user=self.user, isJson=False, params={ 'mimeFilter': json.dumps(['image/png', 'image/jpeg']) }) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Disposition'], 'attachment; filename="Test Collection.zip"') self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) # Only the jpeg should exist in the zip path = 'Test Collection/Test Folder/fake.jpeg' self.assertEqual(zip.namelist(), [path]) extracted = zip.read(path) self.assertEqual(extracted, contents)
def testYAMLConfigFileInherit(server, admin, user, fsAssetstore): # Create some resources to use in the tests collection = Collection().createCollection('collection A', admin) colFolderA = Folder().createFolder(collection, 'folder A', parentType='collection', creator=admin) colFolderB = Folder().createFolder(colFolderA, 'folder B', creator=admin) colFolderConfig = Folder().createFolder(collection, '.config', parentType='collection', creator=admin) collectionB = Collection().createCollection('collection B', admin) configFolder = Folder().createFolder(collectionB, 'any', parentType='collection', creator=admin) Setting().set(constants.PluginSettings.LARGE_IMAGE_CONFIG_FOLDER, str(configFolder['_id'])) utilities.uploadText( json.dumps({ 'keyA': 'value1', 'keyB': 'value2', 'keyC': 'value3', '__inherit__': True }), admin, fsAssetstore, colFolderB, 'sample.json') utilities.uploadText( json.dumps({ 'keyA': 'value4', 'keyD': 'value5', '__inherit__': True }), admin, fsAssetstore, colFolderConfig, 'sample.json') resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=admin) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value1' assert resp.json['keyB'] == 'value2' assert resp.json['keyC'] == 'value3' assert resp.json['keyD'] == 'value5' utilities.uploadText(json.dumps({ 'keyB': 'value6', 'keyE': 'value7' }), admin, fsAssetstore, configFolder, 'sample.json') resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=admin) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value1' assert resp.json['keyB'] == 'value2' assert resp.json['keyC'] == 'value3' assert resp.json['keyD'] == 'value5' assert resp.json['keyE'] == 'value7' Folder().remove(colFolderConfig, user=admin) resp = server.request(path='/folder/%s/yaml_config/sample.json' % str(colFolderB['_id']), user=admin) assert utilities.respStatus(resp) == 200 assert resp.json['keyA'] == 'value1' assert resp.json['keyB'] == 'value2' assert resp.json['keyC'] == 'value3' assert resp.json['keyE'] == 'value7'
def testResourceSearch(self): """ Test resource/search endpoint """ # get expected models from the database admin = User().findOne({'login': '******'}) user = User().findOne({'login': '******'}) coll1 = Collection().findOne({'name': 'Test Collection'}) coll2 = Collection().findOne({'name': 'Magic collection'}) item1 = Item().findOne({'name': 'Public object'}) # set user read permissions on the private collection Collection().setUserAccess(coll2, user, level=AccessType.READ, save=True) # Grab the default user folders resp = self.request(path='/folder', method='GET', user=user, params={ 'parentType': 'user', 'parentId': user['_id'], 'sort': 'name', 'sortdir': 1 }) privateFolder = resp.json[0] # First test all of the required parameters. self.ensureRequiredParams(path='/resource/search', required=['q', 'types']) # Now test parameter validation resp = self.request(path='/resource/search', params={ 'q': 'query', 'types': ',,invalid;json!' }) self.assertStatus(resp, 400) self.assertEqual('Parameter types must be valid JSON.', resp.json['message']) # Test searching with no results resp = self.request(path='/resource/search', params={ 'q': 'gibberish', 'types': '["folder", "user", "collection", "group"]' }) self.assertStatusOk(resp) self.assertEqual(resp.json, { 'folder': [], 'user': [], 'collection': [], 'group': [] }) # Ensure searching respects permissions resp = self.request(path='/resource/search', params={ 'q': 'private', 'types': '["folder", "user", "collection"]' }) self.assertStatusOk(resp) self.assertEqual(resp.json, { 'folder': [], 'user': [], 'collection': [] }) resp = self.request(path='/resource/search', params={ 'q': 'pr', 'mode': 'prefix', 'types': '["folder", "user", "collection"]' }) self.assertStatusOk(resp) self.assertEqual(resp.json, { 'folder': [], 'user': [], 'collection': [] }) resp = self.request(path='/resource/search', params={ 'q': 'private', 'types': '["folder", "user", "collection"]' }, user=user) self.assertStatusOk(resp) self.assertEqual(1, len(resp.json['folder'])) self.assertDictContainsSubset( { '_id': str(privateFolder['_id']), 'name': 'Private' }, resp.json['folder'][0]) self.assertEqual(1, len(resp.json['collection'])) self.assertDictContainsSubset( { '_id': str(coll2['_id']), 'name': coll2['name'] }, resp.json['collection'][0]) self.assertEqual(0, len(resp.json['user'])) resp = self.request(path='/resource/search', params={ 'q': 'pr', 'mode': 'prefix', 'types': '["folder", "user", "collection", "item"]' }, user=user) self.assertStatusOk(resp) self.assertEqual(1, len(resp.json['folder'])) self.assertDictContainsSubset( { '_id': str(privateFolder['_id']), 'name': 'Private' }, resp.json['folder'][0]) self.assertEqual(0, len(resp.json['collection'])) self.assertEqual(0, len(resp.json['item'])) self.assertEqual(0, len(resp.json['user'])) # Ensure that weights are respected, e.g. description should be # weighted less than name. resp = self.request(path='/resource/search', params={ 'q': 'magic', 'types': '["collection"]' }, user=admin) self.assertStatusOk(resp) self.assertEqual(2, len(resp.json['collection'])) self.assertDictContainsSubset( { '_id': str(coll2['_id']), 'name': coll2['name'] }, resp.json['collection'][0]) self.assertDictContainsSubset( { '_id': str(coll1['_id']), 'name': coll1['name'] }, resp.json['collection'][1]) self.assertTrue(resp.json['collection'][0]['_textScore'] > resp.json['collection'][1]['_textScore']) # Exercise user search by login resp = self.request(path='/resource/search', params={ 'q': 'goodlogin', 'types': '["user"]' }, user=admin) self.assertStatusOk(resp) self.assertEqual(1, len(resp.json['user'])) self.assertDictContainsSubset( { '_id': str(user['_id']), 'firstName': user['firstName'], 'lastName': user['lastName'], 'login': user['login'] }, resp.json['user'][0]) # check item search with proper permissions resp = self.request(path='/resource/search', params={ 'q': 'object', 'types': '["item"]' }, user=user) self.assertStatusOk(resp) self.assertEqual(1, len(resp.json['item'])) self.assertDictContainsSubset( { '_id': str(item1['_id']), 'name': item1['name'] }, resp.json['item'][0]) # Check search for model that is not access controlled self.assertNotIsInstance(Assetstore(), AccessControlledModel) self.assertNotIsInstance(Assetstore(), AccessControlMixin) resource.allowedSearchTypes.add('assetstore') resp = self.request(path='/resource/search', params={ 'q': 'Test', 'mode': 'prefix', 'types': '["assetstore"]' }, user=user) self.assertStatusOk(resp) self.assertEqual(1, len(resp.json['assetstore']))
def setUp(self): base.TestCase.setUp(self) # Create a user and a data hierarchy admin = { 'email': '*****@*****.**', 'login': '******', 'firstName': 'Admin', 'lastName': 'Last', 'password': '******', 'admin': True } self.admin = User().createUser(**admin) coll1 = {'name': 'Coll1', 'creator': self.admin} self.coll1 = Collection().createCollection(**coll1) coll2 = {'name': 'Coll2', 'creator': self.admin} self.coll2 = Collection().createCollection(**coll2) folder1 = { 'parent': self.coll1, 'creator': self.admin, 'parentType': 'collection', 'name': 'Top level folder' } self.folder1 = Folder().createFolder(**folder1) folder2 = { 'parent': self.folder1, 'creator': self.admin, 'parentType': 'folder', 'name': 'Subfolder' } self.folder2 = Folder().createFolder(**folder2) item1 = { 'name': 'Item1', 'creator': self.admin, 'folder': self.folder1 } self.item1 = Item().createItem(**item1) item10 = { 'name': 'Item10', 'creator': self.admin, 'folder': self.folder2 } self.item10 = Item().createItem(**item10) file1 = { 'creator': self.admin, 'item': self.item1, 'name': 'File1', 'size': 1, 'assetstore': self.assetstore, 'mimeType': 'text/plain' } self.file1 = File().createFile(**file1) self.file1['sha512'] = '' File().save(self.file1, validate=False) file10 = { 'creator': self.admin, 'item': self.item10, 'name': 'File10', 'size': 10, 'assetstore': self.assetstore, 'mimeType': 'text/plain' } self.file10 = File().createFile(**file10) self.file10['sha512'] = '' File().save(self.file10, validate=False)
def collections(db): yield (Collection().createCollection('private collection', public=False), Collection().createCollection('public collection', public=True))
def testCollectionAccess(self): # Asking to change to an invalid access list should fail resp = self.request(path='/collection/%s/access' % self.collection['_id'], method='PUT', params={ 'access': 'not an access list', 'public': False }, user=self.admin) self.assertStatus(resp, 400) # Create some folders underneath the collection folder1 = Folder().createFolder(parentType='collection', parent=self.collection, creator=self.admin, public=False, name='top level') folder2 = Folder().createFolder(parentType='folder', parent=folder1, creator=self.admin, public=False, name='subfolder') Folder().createFolder(parentType='collection', parent=self.collection, creator=self.admin, public=False, name='another top level folder') # Admin should see two top level folders resp = self.request(path='/collection/%s/details' % self.collection['_id'], user=self.admin) self.assertStatusOk(resp) self.assertEqual(resp.json['nFolders'], 2) self.assertNotIn('nItems', resp.json) # Normal user should see 0 folders resp = self.request(path='/collection/%s/details' % self.collection['_id'], user=self.user) self.assertStatusOk(resp) self.assertEqual(resp.json['nFolders'], 0) # Add read access on one of the folders Folder().setUserAccess(folder1, self.user, AccessType.READ, save=True) # Normal user should see one folder now resp = self.request(path='/collection/%s/details' % self.collection['_id'], user=self.user) self.assertStatusOk(resp) self.assertEqual(resp.json['nFolders'], 1) # Change the access to allow just the user obj = { 'users': [{ 'id': str(self.user['_id']), 'level': AccessType.WRITE }] } resp = self.request(path='/collection/%s/access' % self.collection['_id'], method='PUT', params={ 'access': json.dumps(obj), 'public': True }, user=self.admin) self.assertStatusOk(resp) # Request the collection access resp = self.request(path='/collection/%s/access' % self.collection['_id'], user=self.admin) self.assertStatusOk(resp) access = resp.json self.assertEqual(access['users'][0]['id'], str(self.user['_id'])) self.assertEqual(access['users'][0]['level'], AccessType.WRITE) coll = Collection().load(self.collection['_id'], force=True) folder1 = Folder().load(folder1['_id'], force=True) folder2 = Folder().load(folder2['_id'], force=True) self.assertEqual(coll['public'], True) self.assertEqual(folder1['public'], False) # Update the collection recursively to public resp = self.request(path='/collection/%s/access' % coll['_id'], method='PUT', params={ 'access': json.dumps(obj), 'public': True, 'recurse': True, 'progress': True }, user=self.admin) self.assertStatusOk(resp) coll = Collection().load(coll['_id'], force=True) folder1 = Folder().load(folder1['_id'], force=True) folder2 = Folder().load(folder2['_id'], force=True) self.assertEqual(coll['public'], True) self.assertEqual(folder1['public'], True) self.assertEqual(folder2['public'], True) self.assertEqual(folder1['access'], coll['access']) self.assertEqual(folder1['access'], folder2['access']) self.assertEqual( folder2['access'], { 'users': [{ 'id': self.user['_id'], 'level': AccessType.WRITE, 'flags': [] }], 'groups': [] }) # Recursively drop the user's access level to READ obj['users'][0]['level'] = AccessType.READ resp = self.request(path='/collection/%s/access' % coll['_id'], method='PUT', params={ 'access': json.dumps(obj), 'public': True, 'recurse': True, 'progress': True }, user=self.admin) self.assertStatusOk(resp) coll = Collection().load(coll['_id'], force=True) folder1 = Folder().load(folder1['_id'], force=True) folder2 = Folder().load(folder2['_id'], force=True) self.assertEqual(coll['public'], True) self.assertEqual(folder1['public'], True) self.assertEqual(folder2['public'], True) self.assertEqual(folder1['access'], coll['access']) self.assertEqual(folder1['access'], folder2['access']) self.assertEqual( folder2['access'], { 'users': [{ 'id': self.user['_id'], 'level': AccessType.READ, 'flags': [] }], 'groups': [] }) # Recursively remove the user's access altogether, also make sure that # passing no "public" param just retains the current flag state obj['users'] = () resp = self.request(path='/collection/%s/access' % coll['_id'], method='PUT', params={ 'access': json.dumps(obj), 'recurse': True }, user=self.admin) self.assertStatusOk(resp) coll = Collection().load(coll['_id'], force=True) folder1 = Folder().load(folder1['_id'], force=True) folder2 = Folder().load(folder2['_id'], force=True) self.assertEqual(coll['public'], True) self.assertEqual(folder1['public'], True) self.assertEqual(folder2['public'], True) self.assertEqual(folder1['access'], coll['access']) self.assertEqual(folder1['access'], folder2['access']) self.assertEqual(folder2['access'], {'users': [], 'groups': []}) # Add group access to the collection group = Group().createGroup('test', self.admin) obj = { 'groups': [{ 'id': str(group['_id']), 'level': AccessType.WRITE }] } resp = self.request(path='/collection/%s/access' % coll['_id'], method='PUT', params={ 'access': json.dumps(obj), 'recurse': False }, user=self.admin) self.assertStatusOk(resp) # Create a new top-level folder, it should inherit the collection ACL. resp = self.request(path='/folder', method='POST', params={ 'name': 'top level 2', 'parentId': coll['_id'], 'parentType': 'collection' }, user=self.admin) self.assertStatusOk(resp) folder = Folder().load(resp.json['_id'], force=True) coll = Collection().load(coll['_id'], force=True) self.assertEqual(coll['access']['users'], []) self.assertEqual(folder['access']['users'], [{ 'id': self.admin['_id'], 'level': AccessType.ADMIN, 'flags': [] }]) self.assertEqual(folder['access']['groups'], [{ 'id': group['_id'], 'level': AccessType.WRITE, 'flags': [] }]) self.assertEqual(folder['access']['groups'], coll['access']['groups'])
def testAccessFlags(self): resp = self.request('/system/access_flag') self.assertStatusOk(resp) self.assertEqual(resp.json, {}) registerAccessFlag('my_key', name='hello', description='a custom flag') resp = self.request('/system/access_flag') self.assertStatusOk(resp) self.assertEqual( resp.json, { 'my_key': { 'name': 'hello', 'description': 'a custom flag', 'admin': False } }) self.users[1] = User().load(self.users[1]['_id'], force=True) user = self.users[1] # Manage custom access flags on an access controlled resource self.assertFalse(User().hasAccessFlags(user, user, flags=['my_key'])) # Admin should always have permission self.assertTrue(User().hasAccessFlags(user, self.users[0], flags=['my_key'])) # Test the requireAccessFlags method with self.assertRaises(AccessException): User().requireAccessFlags(user, user=user, flags='my_key') User().requireAccessFlags(user, user=self.users[0], flags='my_key') acl = User().getFullAccessList(user) self.assertEqual(acl['users'][0]['flags'], []) # Test loadmodel requiredFlags argument via REST endpoint resp = self.request('/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1]) self.assertStatus(resp, 403) user = User().setAccessList( self.users[0], access={ 'users': [{ 'id': self.users[1]['_id'], 'level': AccessType.ADMIN, 'flags': ['my_key', 'not a registered flag'] }], 'groups': [{ 'id': self.group['_id'], 'level': AccessType.ADMIN, 'flags': ['my_key'] }] }, save=True) resp = self.request('/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1]) self.assertStatusOk(resp) self.assertEqual(resp.json, 'success') # Only registered flags should be stored acl = User().getFullAccessList(user) self.assertEqual(acl['users'][0]['flags'], ['my_key']) self.assertTrue(User().hasAccessFlags(user, user, flags=['my_key'])) # Create an admin-only access flag registerAccessFlag('admin_flag', name='admin flag', admin=True) # Non-admin shouldn't be able to set it user = User().setAccessList(self.users[0], access={ 'users': [{ 'id': self.users[1]['_id'], 'level': AccessType.ADMIN, 'flags': ['admin_flag'] }], 'groups': [] }, save=True, user=self.users[1]) acl = User().getFullAccessList(user) self.assertEqual(acl['users'][0]['flags'], []) # Admin user should be able to set it user = User().setAccessList(self.users[1], access={ 'users': [{ 'id': self.users[1]['_id'], 'level': AccessType.ADMIN, 'flags': ['admin_flag'] }], 'groups': [{ 'id': self.group['_id'], 'level': AccessType.ADMIN, 'flags': ['admin_flag'] }] }, save=True, user=self.users[0]) acl = User().getFullAccessList(user) self.assertEqual(acl['users'][0]['flags'], ['admin_flag']) # An already-enabled admin-only flag should stay enabled for non-admin user user = User().setAccessList(self.users[1], access={ 'users': [{ 'id': self.users[1]['_id'], 'level': AccessType.ADMIN, 'flags': ['my_key', 'admin_flag'] }], 'groups': [{ 'id': self.group['_id'], 'level': AccessType.ADMIN, 'flags': ['admin_flag'] }] }, save=True, user=self.users[1]) acl = User().getFullAccessList(user) self.assertEqual(set(acl['users'][0]['flags']), {'my_key', 'admin_flag'}) self.assertEqual(acl['groups'][0]['flags'], ['admin_flag']) # Test setting public flags on a collection and folder collectionModel = Collection() folderModel = Folder() itemModel = Item() collection = collectionModel.createCollection('coll', creator=self.users[0], public=True) folder = folderModel.createFolder(collection, 'folder', parentType='collection', creator=self.users[0]) # Add an item to the folder so we can test AclMixin flag behavior item = itemModel.createItem(folder=folder, name='test', creator=self.users[0]) folder = folderModel.setUserAccess(folder, self.users[1], level=AccessType.ADMIN, save=True, currentUser=self.users[0]) with self.assertRaises(AccessException): collectionModel.requireAccessFlags(collection, user=None, flags='my_key') # Test AclMixin flag behavior with self.assertRaises(AccessException): itemModel.requireAccessFlags(item, user=None, flags='my_key') self.assertFalse( itemModel.hasAccessFlags(item, user=None, flags='my_key')) collection = collectionModel.setAccessList(collection, access=collection['access'], save=True, recurse=True, user=self.users[0], publicFlags=['my_key']) collectionModel.requireAccessFlags(collection, user=None, flags='my_key') # Make sure recursive setting of public flags worked folder = folderModel.load(folder['_id'], force=True) self.assertEqual(folder['publicFlags'], ['my_key']) itemModel.requireAccessFlags(item, user=None, flags='my_key') # Non-admin shouldn't be able to set admin-only public flags folder = folderModel.setPublicFlags(folder, flags=['admin_flag'], user=self.users[1], save=True) self.assertEqual(folder['publicFlags'], []) # Admin users should be able to set admin-only public flags folder = folderModel.setPublicFlags(folder, flags=['admin_flag'], user=self.users[0], save=True, append=True) self.assertEqual(folder['publicFlags'], ['admin_flag']) # Non-admin users can set admin-only public flags if they are already enabled folder = folderModel.setPublicFlags(folder, flags=['admin_flag', 'my_key'], user=self.users[1], save=True) self.assertEqual(set(folder['publicFlags']), {'admin_flag', 'my_key'}) # Test "force" options folder = folderModel.setPublicFlags(folder, flags='admin_flag', force=True, save=True) self.assertEqual(folder['publicFlags'], ['admin_flag']) folder = folderModel.setAccessList(folder, access={ 'users': [{ 'id': self.users[1]['_id'], 'level': AccessType.ADMIN, 'flags': ['my_key', 'admin_flag'] }], 'groups': [] }, save=True, force=True) folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key') folder = folderModel.setUserAccess(folder, self.users[1], level=AccessType.READ, save=True, force=True, flags=[]) self.assertFalse( folderModel.hasAccessFlags(folder, self.users[1], flags='my_key')) folder = folderModel.setGroupAccess(folder, self.group, level=AccessType.READ, save=True, force=True, flags='my_key') folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key') # Testing with flags=None should give sensible behavior folderModel.requireAccessFlags(folder, user=None, flags=None) # Test filtering results by access flags (both ACModel and AclMixin) for model, doc in ((folderModel, folder), (itemModel, item)): cursor = model.find({}) self.assertGreater(len(list(cursor)), 0) cursor = model.find({}) filtered = list( model.filterResultsByPermission(cursor, user=None, level=AccessType.READ, flags='my_key')) self.assertEqual(len(filtered), 0) cursor = model.find({}) filtered = list( model.filterResultsByPermission(cursor, user=self.users[1], level=AccessType.READ, flags=('my_key', 'admin_flag'))) self.assertEqual(len(filtered), 1) self.assertEqual(filtered[0]['_id'], doc['_id'])
def testEmptyCreator(self): c = Collection().createCollection('No Creator') self.assertEqual(c['creatorId'], None)
def _createFiles(self, user=None): """ Create a set of items, folders, files, metadata, and collections for testing. :param user: the user who should own these items. """ if user is None: user = self.admin self.expectedZip = {} # Create a collection coll = { 'name': 'Test Collection', 'description': 'The description', 'public': True, 'creator': user } self.collection = Collection().createCollection(**coll) self.collectionPrivateFolder = Folder().createFolder( parent=self.collection, parentType='collection', name='Private', creator=user, public=False) # Get the admin user's folders resp = self.request(path='/folder', method='GET', user=user, params={ 'parentType': 'user', 'parentId': user['_id'], 'sort': 'name', 'sortdir': 1 }) self.adminPrivateFolder = Folder().load(resp.json[0]['_id'], user=user) self.adminPublicFolder = Folder().load(resp.json[1]['_id'], user=user) # Create a folder within the admin public folder resp = self.request(path='/folder', method='POST', user=user, params={ 'name': 'Folder 1', 'parentId': self.adminPublicFolder['_id'] }) self.adminSubFolder = resp.json # Create a series of items self.items = [] self.items.append(Item().createItem('Item 1', self.admin, self.adminPublicFolder)) self.items.append(Item().createItem('Item 2', self.admin, self.adminPublicFolder)) self.items.append(Item().createItem('It\\em/3', self.admin, self.adminSubFolder)) self.items.append(Item().createItem('Item 4', self.admin, self.collectionPrivateFolder)) self.items.append(Item().createItem('Item 5', self.admin, self.collectionPrivateFolder)) # Upload a series of files file, path, contents = self._uploadFile('File 1', self.items[0]) self.file1 = file self.expectedZip[path] = contents file, path, contents = self._uploadFile('File 2', self.items[0]) self.expectedZip[path] = contents file, path, contents = self._uploadFile('File 3', self.items[1]) self.expectedZip[path] = contents file, path, contents = self._uploadFile('File 4', self.items[2]) self.expectedZip[path] = contents file, path, contents = self._uploadFile('File 5', self.items[3]) self.expectedZip[path] = contents # place some metadata on two of the items and one of the folders meta = {'key': 'value'} Item().setMetadata(self.items[2], meta) parents = Item().parentsToRoot(self.items[2], self.admin) path = os.path.join(*([ part['object'].get('name', part['object'].get('login', '')) for part in parents ] + [self.items[2]['name'], 'girder-item-metadata.json'])) self.expectedZip[path] = meta meta = {'x': 'y'} Item().setMetadata(self.items[4], meta) parents = Item().parentsToRoot(self.items[4], self.admin) path = os.path.join(*([ part['object'].get('name', part['object'].get('login', '')) for part in parents ] + [self.items[4]['name'], 'girder-item-metadata.json'])) self.expectedZip[path] = meta meta = {'key2': 'value2', 'date': datetime.datetime.utcnow()} # mongo rounds to millisecond, so adjust our expectations meta['date'] -= datetime.timedelta( microseconds=meta['date'].microsecond % 1000) Folder().setMetadata(self.adminPublicFolder, meta) parents = Folder().parentsToRoot(self.adminPublicFolder, user=user) path = os.path.join(*([ part['object'].get('name', part['object'].get('login', '')) for part in parents ] + [self.adminPublicFolder['name'], 'girder-folder-metadata.json'])) self.expectedZip[path] = meta
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'): """ As needed, download sample data. :param adminUser: a user to create and modify collections and folders. :param collName: the collection name where the data will be added. :param folderName: the folder name where the data will bed aded. :returns: the folder where the sample data is located. """ if Collection().findOne({'lowerName': collName.lower()}) is None: Collection().createCollection(collName, adminUser) collection = Collection().findOne({'lowerName': collName.lower()}) if Folder().findOne({ 'parentId': collection['_id'], 'lowerName': folderName.lower() }) is None: Folder().createFolder(collection, folderName, parentType='collection', public=True, creator=adminUser) folder = Folder().findOne({ 'parentId': collection['_id'], 'lowerName': folderName.lower() }) remote = girder_client.GirderClient( apiUrl='https://data.kitware.com/api/v1') remoteFolder = remote.resourceLookup( '/collection/HistomicsTK/Deployment test images') sampleItems = [] for remoteItem in remote.listItem(remoteFolder['_id']): item = Item().findOne({ 'folderId': folder['_id'], 'name': remoteItem['name'] }) if item and len(list(Item().childFiles(item, limit=1))): sampleItems.append(item) continue if not item: item = Item().createItem(remoteItem['name'], creator=adminUser, folder=folder) for remoteFile in remote.listFile(remoteItem['_id']): with tempfile.NamedTemporaryFile() as tf: fileName = tf.name tf.close() sys.stdout.write('Downloading %s' % remoteFile['name']) sys.stdout.flush() remote.downloadFile(remoteFile['_id'], fileName) sys.stdout.write(' .') sys.stdout.flush() Upload().uploadFromFile(open(fileName, 'rb'), os.path.getsize(fileName), name=remoteItem['name'], parentType='item', parent=item, user=adminUser) sys.stdout.write('.\n') sys.stdout.flush() sampleItems.append(item) for item in sampleItems: if 'largeImage' not in item: sys.stdout.write('Making large_item %s ' % item['name']) sys.stdout.flush() try: ImageItem().createImageItem(item, createJob=False) except Exception: pass print('done') return folder