def __init__(self):
super(Collection, self).__init__()
self.resourceName = 'collection'
self._model = CollectionModel()
self.route('DELETE', (':id',), self.deleteCollection)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getCollection)
self.route('GET', (':id', 'details'), self.getCollectionDetails)
self.route('GET', (':id', 'download'), self.downloadCollection)
self.route('GET', (':id', 'access'), self.getCollectionAccess)
self.route('POST', (), self.createCollection)
self.route('PUT', (':id',), self.updateCollection)
self.route('PUT', (':id', 'access'), self.updateCollectionAccess)
class Collection(Resource):
"""API Endpoint for collections."""
def __init__(self):
super(Collection, self).__init__()
self.resourceName = 'collection'
self._model = CollectionModel()
self.route('DELETE', (':id',), self.deleteCollection)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getCollection)
self.route('GET', (':id', 'details'), self.getCollectionDetails)
self.route('GET', (':id', 'download'), self.downloadCollection)
self.route('GET', (':id', 'access'), self.getCollectionAccess)
self.route('POST', (), self.createCollection)
self.route('PUT', (':id',), self.updateCollection)
self.route('PUT', (':id', 'access'), self.updateCollectionAccess)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('List or search for collections.')
.responseClass('Collection', array=True)
.param('text', 'Pass this to perform a text search for collections.', required=False)
.pagingParams(defaultSort='name')
)
def find(self, text, limit, offset, sort):
user = self.getCurrentUser()
if text is not None:
return self._model.textSearch(text, user=user, limit=limit, offset=offset)
return self._model.list(user=user, offset=offset, limit=limit, sort=sort)
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Create a new collection.')
.responseClass('Collection')
.param('name', 'Name for the collection. Must be unique.')
.param('description', 'Collection description.', required=False)
.param('public', 'Whether the collection should be publicly visible.',
required=False, dataType='boolean', default=False)
.errorResponse()
.errorResponse('You are not authorized to create collections.', 403)
)
def createCollection(self, name, description, public):
user = self.getCurrentUser()
if not self._model.hasCreatePrivilege(user):
raise AccessException('You are not authorized to create collections.')
return self._model.createCollection(
name=name, description=description, public=public, creator=user)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Get a collection by ID.')
.responseClass('Collection')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read permission denied on the collection.', 403)
)
def getCollection(self, collection):
return collection
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Get detailed information about a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.errorResponse()
.errorResponse('Read access was denied on the collection.', 403)
)
def getCollectionDetails(self, collection):
return {
'nFolders': self._model.countFolders(
collection, user=self.getCurrentUser(), level=AccessType.READ)
}
@access.cookie
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Download an entire collection as a zip archive.')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.jsonParam('mimeFilter', 'JSON list of MIME types to include.', requireArray=True,
required=False)
.produces('application/zip')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the collection.', 403)
)
def downloadCollection(self, collection, mimeFilter):
setResponseHeader('Content-Type', 'application/zip')
setContentDisposition(collection['name'] + '.zip')
def stream():
zip = ziputil.ZipGenerator(collection['name'])
for (path, file) in self._model.fileList(
collection, user=self.getCurrentUser(), subpath=False, mimeFilter=mimeFilter):
for data in zip.addFile(file, path):
yield data
yield zip.footer()
return stream
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Get the access control list for a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def getCollectionAccess(self, collection):
return self._model.getFullAccessList(collection)
@access.user(scope=TokenScope.DATA_OWN)
@filtermodel(model=CollectionModel, addFields={'access'})
@autoDescribeRoute(
Description('Set the access control list for a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.jsonParam('access', 'The access control list as JSON.', requireObject=True)
.jsonParam('publicFlags', 'List of public access flags to set on the collection.',
required=False, requireArray=True)
.param('public', 'Whether the collection should be publicly visible.',
dataType='boolean', required=False)
.param('recurse', 'Whether the policies should be applied to all '
'folders under this collection as well.', dataType='boolean',
default=False, required=False)
.param('progress', 'If recurse is set to True, this controls whether '
'progress notifications will be sent.', dataType='boolean',
default=False, required=False)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def updateCollectionAccess(self, collection, access, public, recurse, progress, publicFlags):
user = self.getCurrentUser()
progress = progress and recurse
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self._model.subtreeCount(
collection, includeItems=False, user=user, level=AccessType.ADMIN))
return self._model.setAccessList(
collection, access, save=True, user=user, recurse=recurse,
progress=ctx, setPublic=public, publicFlags=publicFlags)
@access.user(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Edit a collection by ID.')
.responseClass('Collection')
.modelParam('id', model=CollectionModel, level=AccessType.WRITE)
.param('name', 'Unique name for the collection.', required=False, strip=True)
.param('description', 'Collection description.', required=False, strip=True)
.errorResponse('ID was invalid.')
.errorResponse('Write permission denied on the collection.', 403)
)
def updateCollection(self, collection, name, description):
if name is not None:
collection['name'] = name
if description is not None:
collection['description'] = description
return self._model.updateCollection(collection)
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Delete a collection by ID.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def deleteCollection(self, collection):
self._model.remove(collection)
return {'message': 'Deleted collection %s.' % collection['name']}
def testConsistencyCheck(self):
user = self.users[0]
c1 = Collection().createCollection('c1', user)
f1 = Folder().createFolder(c1, 'f1', parentType='collection')
Folder().createFolder(c1, 'f2', parentType='collection')
f3 = Folder().createFolder(user, 'f3', parentType='user')
Folder().createFolder(user, 'f4', parentType='user')
i1 = Item().createItem('i1', user, f1)
i2 = Item().createItem('i2', user, f1)
Item().createItem('i3', user, f1)
i4 = Item().createItem('i4', user, f3)
Item().createItem('i5', user, f3)
Item().createItem('i6', user, f3)
assetstore = {'_id': 0}
File().createFile(user, i1, 'foo', 7, assetstore)
File().createFile(user, i1, 'foo', 13, assetstore)
File().createFile(user, i2, 'foo', 19, assetstore)
File().createFile(user, i4, 'foo', 23, assetstore)
self.assertEqual(39, Collection().load(c1['_id'], force=True)['size'])
self.assertEqual(39, Folder().load(f1['_id'], force=True)['size'])
self.assertEqual(23, Folder().load(f3['_id'], force=True)['size'])
self.assertEqual(20, Item().load(i1['_id'], force=True)['size'])
self.assertEqual(23, User().load(user['_id'], force=True)['size'])
resp = self.request(path='/system/check', user=user, method='PUT')
self.assertStatusOk(resp)
self.assertEqual(resp.json['baseParentsFixed'], 0)
self.assertEqual(resp.json['orphansRemoved'], 0)
self.assertEqual(resp.json['sizesChanged'], 0)
Item().update({'_id': i1['_id']},
update={'$set': {
'baseParentId': None
}})
resp = self.request(path='/system/check', user=user, method='PUT')
self.assertStatusOk(resp)
self.assertEqual(resp.json['baseParentsFixed'], 1)
self.assertEqual(resp.json['orphansRemoved'], 0)
self.assertEqual(resp.json['sizesChanged'], 0)
Collection().update({'_id': c1['_id']}, update={'$set': {'size': 0}})
Folder().update({'_id': f1['_id']}, update={'$set': {'size': 0}})
Item().update({'_id': i1['_id']}, update={'$set': {'size': 0}})
resp = self.request(path='/system/check', user=user, method='PUT')
self.assertStatusOk(resp)
self.assertEqual(resp.json['baseParentsFixed'], 0)
self.assertEqual(resp.json['orphansRemoved'], 0)
self.assertEqual(resp.json['sizesChanged'], 3)
self.assertEqual(39, Collection().load(c1['_id'], force=True)['size'])
self.assertEqual(39, Folder().load(f1['_id'], force=True)['size'])
self.assertEqual(23, Folder().load(f3['_id'], force=True)['size'])
self.assertEqual(20, Item().load(i1['_id'], force=True)['size'])
self.assertEqual(23, User().load(user['_id'], force=True)['size'])
Folder().collection.delete_one({'_id': f3['_id']})
resp = self.request(path='/system/check', user=user, method='PUT')
self.assertStatusOk(resp)
self.assertEqual(resp.json['baseParentsFixed'], 0)
self.assertEqual(resp.json['orphansRemoved'], 3)
self.assertEqual(resp.json['sizesChanged'], 0)
self.assertEqual(0, User().load(user['_id'], force=True)['size'])
def lookUpPath(path, user=None, test=False, filter=True, force=False):
"""
Look up a resource in the data hierarchy by path.
:param path: path of the resource
:param user: user with correct privileges to access path
:param test: defaults to false, when set to true
will return None instead of throwing exception when
path doesn't exist
:type test: bool
:param filter: Whether the returned model should be filtered.
:type filter: bool
:param force: if True, don't validate the access.
:type force: bool
"""
path = path.lstrip('/')
pathArray = split(path)
model = pathArray[0]
if model == 'user':
username = pathArray[1]
parent = User().findOne({'login': username})
if parent is None:
if test:
return {'model': None, 'document': None}
else:
raise ResourcePathNotFound('User not found: %s' % username)
elif model == 'collection':
collectionName = pathArray[1]
parent = Collection().findOne({'name': collectionName})
if parent is None:
if test:
return {'model': None, 'document': None}
else:
raise ResourcePathNotFound('Collection not found: %s' %
collectionName)
else:
raise ValidationException('Invalid path format')
try:
document = parent
if not force:
ModelImporter.model(model).requireAccess(document, user)
for token in pathArray[2:]:
document, model = lookUpToken(token, model, document)
if not force:
ModelImporter.model(model).requireAccess(document, user)
except (ValidationException, AccessException):
# We should not distinguish the response between access and validation errors so that
# adversarial users cannot discover the existence of data they don't have access to by
# looking up a path.
if test:
return {'model': None, 'document': None}
else:
raise ResourcePathNotFound('Path not found: %s' % path)
if filter:
document = ModelImporter.model(model).filter(document, user)
return {'model': model, 'document': document}
def loadStudyCollection(self):
# assumes collection has been created by provision_utility
# TODO: cache this value
return Collection().findOne({'name': 'Annotation Studies'})
def testDownload(self):
collection = Collection().createCollection('collection1', public=True)
folder = Folder().createFolder(collection,
'folder1',
parentType='collection',
public=True)
item = Item().createItem('item1', self.admin, folder)
# Path to test files
file1Path = os.path.join(self.filesDir, 'txt1.txt')
file2Path = os.path.join(self.filesDir, 'txt2.txt')
# Upload files to item
with open(file1Path, 'rb') as fp:
file1 = Upload().uploadFromFile(fp,
os.path.getsize(file1Path),
'txt1.txt',
parentType='item',
parent=item,
user=self.admin)
with open(file2Path, 'rb') as fp:
file2 = Upload().uploadFromFile(fp,
os.path.getsize(file2Path),
'txt2.txt',
mimeType='image/jpeg',
parentType='item',
parent=item,
user=self.admin)
# Download item and its files several times and ensure downloads are recorded
# Each file is downloaded 10 times
for _ in range(0, 5):
self._downloadItem(item['_id'])
self._downloadFile(file1['_id'])
self._downloadFile(file2['_id'])
# Download each file 1 time by downloading parent folder
self._downloadFolder(folder['_id'])
# Download each file over 2 requests
self._downloadFileInTwoChunks(file1['_id'])
self._downloadFileInTwoChunks(file2['_id'])
# Download each file partially, adding 1 to start and 4 to requested
self._downloadPartialFile(file1['_id'])
self._downloadPartialFile(file2['_id'])
# Download entire collection
# Each file is downloaded 1 additional time
path = '/collection/%s/download' % collection['_id']
resp = self.request(path, user=self.admin, isJson=False)
# Iterate through generator to trigger download events
for data in resp.body:
data
# Download collection filtered by mime type
# file2 is downloaded one additional time
path = '/collection/%s/download' % collection['_id']
resp = self.request(path,
user=self.admin,
isJson=False,
method='GET',
params={
'id': collection['_id'],
'mimeFilter': json.dumps(['image/jpeg'])
})
# iterate through generator to trigger download events
for data in resp.body:
data
self._checkDownloadsCount(file1['_id'], 14, 18, 13)
self._checkDownloadsCount(file2['_id'], 15, 19, 14)
def createDataset(self, name, description, license, attribution, owner, creatorUser):
now = datetime.datetime.utcnow()
# Validate before saving anything
dataset = self.save({
# Public informational data
'name': name,
'description': description,
'license': license,
'attribution': attribution,
# Public Girder data
'created': now,
'updated': now,
# Private informational data
'owner': owner,
'metadataFiles': [],
# Private Girder data
'creatorId': creatorUser['_id'],
'folderId': None,
'public': False,
'access': {
'users': [],
'groups': []
}
})
# Create folder and add it to the dataset
datasetFolder = Folder().createFolder(
parent=Collection().findOne({'name': 'Lesion Images'}),
name=dataset['name'],
parentType='collection',
creator=creatorUser,
public=dataset['public'],
allowRename=False,
reuseExisting=False)
dataset['folderId'] = datasetFolder['_id']
self.update(
{'_id': dataset['_id']},
{'$set': {'folderId': dataset['folderId']}}
)
# Set default accesses (overwriting inherited accesses on the folder)
dataset = self.setAccessList(
doc=dataset,
access={
'users': [
# Allow the creator to write
{
'id': creatorUser['_id'],
'level': AccessType.WRITE,
'flags': []
}
],
'groups': [
# Allow reviewers to admin (so they can delete the dataset)
{
'id': Group().findOne({'name': 'Dataset QC Reviewers'})['_id'],
'level': AccessType.ADMIN,
'flags': []
}
]
}
)
return dataset
class Collection(Resource):
"""API Endpoint for collections."""
def __init__(self):
super(Collection, self).__init__()
self.resourceName = 'collection'
self._model = CollectionModel()
self.route('DELETE', (':id',), self.deleteCollection)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getCollection)
self.route('GET', (':id', 'details'), self.getCollectionDetails)
self.route('GET', (':id', 'download'), self.downloadCollection)
self.route('GET', (':id', 'access'), self.getCollectionAccess)
self.route('POST', (), self.createCollection)
self.route('PUT', (':id',), self.updateCollection)
self.route('PUT', (':id', 'access'), self.updateCollectionAccess)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('List or search for collections.')
.responseClass('Collection', array=True)
.param('text', 'Pass this to perform a text search for collections.', required=False)
.pagingParams(defaultSort='name')
)
def find(self, text, limit, offset, sort):
user = self.getCurrentUser()
if text is not None:
return self._model.textSearch(text, user=user, limit=limit, offset=offset)
return self._model.list(user=user, offset=offset, limit=limit, sort=sort)
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Create a new collection.')
.responseClass('Collection')
.param('name', 'Name for the collection. Must be unique.')
.param('description', 'Collection description.', required=False)
.param('public', 'Whether the collection should be publicly visible.',
required=False, dataType='boolean', default=False)
.errorResponse()
.errorResponse('You are not authorized to create collections.', 403)
)
def createCollection(self, name, description, public):
user = self.getCurrentUser()
if not self._model.hasCreatePrivilege(user):
raise AccessException('You are not authorized to create collections.')
return self._model.createCollection(
name=name, description=description, public=public, creator=user)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Get a collection by ID.')
.responseClass('Collection')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read permission denied on the collection.', 403)
)
def getCollection(self, collection):
return collection
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Get detailed information about a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.errorResponse()
.errorResponse('Read access was denied on the collection.', 403)
)
def getCollectionDetails(self, collection):
return {
'nFolders': self._model.countFolders(
collection, user=self.getCurrentUser(), level=AccessType.READ)
}
@access.public(scope=TokenScope.DATA_READ, cookie=True)
@autoDescribeRoute(
Description('Download an entire collection as a zip archive.')
.modelParam('id', model=CollectionModel, level=AccessType.READ)
.jsonParam('mimeFilter', 'JSON list of MIME types to include.', requireArray=True,
required=False)
.produces('application/zip')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the collection.', 403)
)
def downloadCollection(self, collection, mimeFilter):
setResponseHeader('Content-Type', 'application/zip')
setContentDisposition(collection['name'] + '.zip')
def stream():
zip = ziputil.ZipGenerator(collection['name'])
for (path, file) in self._model.fileList(
collection, user=self.getCurrentUser(), subpath=False, mimeFilter=mimeFilter):
for data in zip.addFile(file, path):
yield data
yield zip.footer()
return stream
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Get the access control list for a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def getCollectionAccess(self, collection):
return self._model.getFullAccessList(collection)
@access.user(scope=TokenScope.DATA_OWN)
@filtermodel(model=CollectionModel, addFields={'access'})
@autoDescribeRoute(
Description('Set the access control list for a collection.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.jsonParam('access', 'The access control list as JSON.', requireObject=True)
.jsonParam('publicFlags', 'List of public access flags to set on the collection.',
required=False, requireArray=True)
.param('public', 'Whether the collection should be publicly visible.',
dataType='boolean', required=False)
.param('recurse', 'Whether the policies should be applied to all '
'folders under this collection as well.', dataType='boolean',
default=False, required=False)
.param('progress', 'If recurse is set to True, this controls whether '
'progress notifications will be sent.', dataType='boolean',
default=False, required=False)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def updateCollectionAccess(self, collection, access, public, recurse, progress, publicFlags):
user = self.getCurrentUser()
progress = progress and recurse
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self._model.subtreeCount(
collection, includeItems=False, user=user, level=AccessType.ADMIN))
return self._model.setAccessList(
collection, access, save=True, user=user, recurse=recurse,
progress=ctx, setPublic=public, publicFlags=publicFlags)
@access.user(scope=TokenScope.DATA_READ)
@filtermodel(model=CollectionModel)
@autoDescribeRoute(
Description('Edit a collection by ID.')
.responseClass('Collection')
.modelParam('id', model=CollectionModel, level=AccessType.WRITE)
.param('name', 'Unique name for the collection.', required=False, strip=True)
.param('description', 'Collection description.', required=False, strip=True)
.errorResponse('ID was invalid.')
.errorResponse('Write permission denied on the collection.', 403)
)
def updateCollection(self, collection, name, description):
if name is not None:
collection['name'] = name
if description is not None:
collection['description'] = description
return self._model.updateCollection(collection)
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Delete a collection by ID.')
.modelParam('id', model=CollectionModel, level=AccessType.ADMIN)
.errorResponse('ID was invalid.')
.errorResponse('Admin permission denied on the collection.', 403)
)
def deleteCollection(self, collection):
self._model.remove(collection)
return {'message': 'Deleted collection %s.' % collection['name']}
def getTCGACollection(self):
"""Get the unique TCGA collection from the settings collection."""
tcga = Setting().get(TCGACollectionSettingKey)
if tcga is None:
raise Exception('TCGA collection id not initialized in settings')
return Collection().load(tcga, force=True)
def _validateStudiesColl(doc):
Collection().load(doc['value'], exc=True, force=True)
def setUp(self):
super(ResourceOperationsTestCase, self).setUp()
users = (
{
"email": "*****@*****.**",
"login": "******",
"firstName": "Jane",
"lastName": "Austin",
"password": "******",
},
{
"email": "*****@*****.**",
"login": "******",
"firstName": "Sally",
"lastName": "User",
"password": "******",
},
)
self.users = {
user["login"]: User().createUser(**user)
for user in users
}
self.public_root = tempfile.mkdtemp()
self.private_root = tempfile.mkdtemp()
self.base_collection = Collection().createCollection(
"Virtual Resources",
creator=self.users["admin"],
public=True,
reuseExisting=True,
)
self.public_folder = Folder().createFolder(
self.base_collection,
"public",
parentType="collection",
public=True,
reuseExisting=True,
)
self.public_folder.update(dict(fsPath=self.public_root,
isMapping=True))
self.public_folder = Folder().save(self.public_folder)
self.regular_folder = Folder().createFolder(
self.base_collection,
"public_no_map",
parentType="collection",
public=True,
reuseExisting=True,
)
self.private_folder = Folder().createFolder(
self.base_collection,
"private",
creator=self.users["sally"],
parentType="collection",
public=False,
reuseExisting=True,
)
self.private_folder.update(
dict(fsPath=self.private_root, isMapping=True))
self.private_folder = Folder().save(self.private_folder)
def setUp(self):
super(SlicerExtensionManagerTest, self).setUp()
self.maxDiff = None
self._MAX_CHUNK_SIZE = 1024 * 1024 * 64
self._extensionNameTemplate = '{app_revision}_{os}_{arch}_{baseName}_{revision}'
self.dataDir = os.path.join(os.environ['GIRDER_TEST_DATA_PREFIX'],
'plugins', 'slicer_extension_manager')
self._user = User().createUser('usr0', 'passwd', 'tst', 'usr',
'*****@*****.**')
self._collection = Collection().createCollection(
'testCollection',
creator=self._user,
description='Contain applications')
self._app = Folder().createFolder(parent=self._collection,
name='application',
description='app description',
parentType='Collection',
public=True,
creator=self._user)
self._app = Folder().setMetadata(
self._app, {'extensionNameTemplate': self._extensionNameTemplate})
self._nightly = Folder().createFolder(
parent=self._app,
name=constants.NIGHTLY_RELEASE_NAME,
description='Uploaded each night, always up-to-date',
parentType='Folder',
public=True,
creator=self._user)
self._release = Folder().createFolder(
parent=self._app,
name='release',
description='release description',
parentType='Folder',
public=True,
creator=self._user)
self._release = Folder().setMetadata(self._release,
{'revision': '12345'})
self._extensions = {
'extension1': {
'name': '12345_linux_i386_slicerExt_35333',
'meta': {
'os': 'linux',
'arch': 'i386',
'baseName': 'slicerExt',
'repository_type': 'git',
'repository_url': 'http://slicer.com/extension/Ext',
'revision': '35333',
'app_revision': '12345',
'packagetype': 'installer',
'codebase': 'SL4',
'description': 'Extension for Slicer 4'
}
},
'extension2': {
'name': '112233_win_i386_slicerExt1_54342',
'meta': {
'os': 'win',
'arch': 'i386',
'baseName': 'slicerExt1',
'repository_type': 'git',
'repository_url': 'http://slicer.com/extension/Ext',
'revision': '54342',
'app_revision': '112233',
'packagetype': 'installer',
'codebase': 'SL4',
'description': 'Extension for Slicer 4 new version'
}
},
'extension3': {
'name': '112233_linux_amd64_slicerExt2_542',
'meta': {
'os': 'linux',
'arch': 'amd64',
'baseName': 'slicerExt2',
'repository_type': 'gitlab',
'repository_url': 'http://slicer.com/extension/Ext',
'revision': '542',
'app_revision': '112233',
'packagetype': 'zip',
'codebase': 'SL434334',
'description': 'Extension for Slicer 4 new version'
}
}
}
def testMongoSearch(self):
"""
Test resource/mongo_search endpoint
"""
# Create a bunch of searchable documents
admin = {
'email': '*****@*****.**',
'login': '******',
'firstName': 'Admin',
'lastName': 'Last',
'password': '******',
'admin': True
}
admin = User().createUser(**admin)
user = {
'email': '*****@*****.**',
'login': '******',
'firstName': 'First',
'lastName': 'Last',
'password': '******',
'admin': False
}
user = User().createUser(**user)
coll1 = {
'name': 'Test Collection',
'description': 'magic words. And more magic.',
'public': True,
'creator': admin
}
coll1 = Collection().createCollection(**coll1)
coll2 = {
'name': 'Magic collection',
'description': 'private',
'public': False,
'creator': admin
}
coll2 = Collection().createCollection(**coll2)
Collection().setUserAccess(coll2,
user,
level=AccessType.READ,
save=True)
folder1 = {
'parent': coll1,
'parentType': 'collection',
'name': 'Public test folder'
}
folder1 = Folder().createFolder(**folder1)
Folder().setUserAccess(folder1,
user,
level=AccessType.READ,
save=False)
Folder().setPublic(folder1, True, save=True)
folder2 = {
'parent': coll2,
'parentType': 'collection',
'name': 'Private test folder'
}
folder2 = Folder().createFolder(**folder2)
Folder().setUserAccess(folder2, user, level=AccessType.NONE, save=True)
item1 = {'name': 'Public object', 'creator': admin, 'folder': folder1}
item1 = Item().createItem(**item1)
item2 = {'name': 'Secret object', 'creator': admin, 'folder': folder2}
item2 = Item().createItem(**item2)
# Grab the default user folders
resp = self.request(path='/folder',
method='GET',
user=user,
params={
'parentType': 'user',
'parentId': user['_id'],
'sort': 'name',
'sortdir': 1
})
# First test all of the required parameters.
self.ensureRequiredParams(path='/resource/search',
required=['q', 'types'])
# Now test parameter validation
resp = self.request(path='/resource/mongo_search',
params={
'q': 'query',
'type': 'wrong type'
})
self.assertStatus(resp, 400)
self.assertEqual('Invalid resource type: wrong type',
resp.json['message'])
# Test validation of JSON input
resp = self.request(path='/resource/mongo_search',
params={
'q': 'not_json',
'type': 'folder'
})
self.assertStatus(resp, 400)
self.assertEqual(resp.json['message'],
'The query parameter must be a JSON object.')
# Ensure searching respects permissions
resp = self.request(path='/resource/mongo_search',
params={
'q': bson.json_util.dumps({'name': 'Private'}),
'type': 'folder'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
resp = self.request(path='/resource/mongo_search',
params={
'q': bson.json_util.dumps({'name': 'Private'}),
'type': 'folder'
},
user=user)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertHasKeys(resp.json[0], ('_id', 'name', 'description'))
self.assertEqual(len(resp.json[0]), 3)
# Test item search
resp = self.request(path='/resource/mongo_search',
params={
'q':
bson.json_util.dumps(
{'folderId': folder1['_id']}),
'type':
'item'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, [{
'_id': str(item1['_id']),
'name': 'Public object',
'description': '',
'folderId': str(folder1['_id'])
}])
resp = self.request(path='/resource/mongo_search',
params={
'q':
bson.json_util.dumps(
{'folderId': folder2['_id']}),
'type':
'item'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, [])
resp = self.request(path='/resource/mongo_search',
params={
'q':
bson.json_util.dumps(
{'folderId': folder2['_id']}),
'type':
'item'
},
user=admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json, [{
'_id': str(item2['_id']),
'name': 'Secret object',
'description': '',
'folderId': str(folder2['_id'])
}])
def testResourceItems(self):
# Create some resources to use in the tests
self.collection = Collection().createCollection(
'collection A', self.admin)
self.colFolderA = Folder().createFolder(self.collection,
'folder A',
parentType='collection',
creator=self.admin)
self.colFolderB = Folder().createFolder(self.collection,
'folder B',
parentType='collection',
creator=self.admin)
self.colFolderC = Folder().createFolder(self.colFolderA,
'folder C',
creator=self.admin)
self.colItemA1 = Item().createItem('item A1', self.admin,
self.colFolderA)
self.colItemB1 = Item().createItem('item B1', self.admin,
self.colFolderB)
self.colItemB2 = Item().createItem('item B2', self.admin,
self.colFolderB)
self.colItemC1 = Item().createItem('item C1', self.admin,
self.colFolderC)
self.colItemC2 = Item().createItem('item C2', self.admin,
self.colFolderC)
self.colItemC3 = Item().createItem('item C3', self.admin,
self.colFolderC)
self.itemPub1 = Item().createItem('item Public 1', self.admin,
self.publicFolder)
self.itemPriv1 = Item().createItem('item Private 1', self.admin,
self.privateFolder)
self.folderD = Folder().createFolder(self.publicFolder,
'folder D',
creator=self.admin)
self.itemD1 = Item().createItem('item D1', self.admin, self.folderD)
self.itemD2 = Item().createItem('item D2', self.admin, self.folderD)
# Now test that we get the items we expect
# From a user
resp = self.request(path='/resource/%s/items' % self.admin['_id'],
user=self.admin,
params={'type': 'user'})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual(
[item['name'] for item in items],
['item Public 1', 'item D1', 'item D2', 'item Private 1'])
# From a collection
resp = self.request(path='/resource/%s/items' % self.collection['_id'],
user=self.admin,
params={'type': 'collection'})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual(
[item['name'] for item in items],
['item A1', 'item C1', 'item C2', 'item C3', 'item B1', 'item B2'])
# With sort, limit, and offset
resp = self.request(path='/resource/%s/items' % self.collection['_id'],
user=self.admin,
params={
'type': 'collection',
'limit': 4,
'offset': 1,
'sort': 'name',
'sortdir': -1
})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual([item['name'] for item in items],
['item B1', 'item A1', 'item C3', 'item C2'])
resp = self.request(path='/resource/%s/items' % self.collection['_id'],
user=self.admin,
params={
'type': 'collection',
'limit': 1,
'offset': 0,
'sort': 'name',
'sortdir': -1
})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual([item['name'] for item in items], ['item B2'])
# From a folder
resp = self.request(path='/resource/%s/items' % self.colFolderA['_id'],
user=self.admin,
params={'type': 'folder'})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual([item['name'] for item in items],
['item A1', 'item C1', 'item C2', 'item C3'])
# From a lower folder
resp = self.request(path='/resource/%s/items' % self.colFolderC['_id'],
user=self.admin,
params={'type': 'folder'})
self.assertStatusOk(resp)
items = resp.json
self.assertEqual([item['name'] for item in items],
['item C1', 'item C2', 'item C3'])
# With a bad parameter
resp = self.request(path='/resource/%s/items' % self.colFolderC['_id'],
user=self.admin,
params={'type': 'collection'})
self.assertStatus(resp, 400)
self.assertIn('Resource not found', resp.json['message'])
def testResourceMetadata(self):
# Create some resources to use in the tests
self.collection = Collection().createCollection(
'collection A', self.admin)
self.colFolderA = Folder().createFolder(self.collection,
'folder A',
parentType='collection',
creator=self.admin)
self.colItemA1 = Item().createItem('item A1', self.admin,
self.colFolderA)
self.colItemB1 = Item().createItem('item B1', self.admin,
self.colFolderA)
resp = self.request(method='PUT',
path='/resource/metadata',
params={
'resources':
json.dumps(
{'item': [str(self.colItemA1['_id'])]}),
'metadata':
json.dumps({
'keya': 'valuea',
'keyb.keyc': 'valuec'
})
})
self.assertStatus(resp, 401)
resp = self.request(method='PUT',
path='/resource/metadata',
user=self.admin,
params={
'resources':
json.dumps(
{'item': [str(self.colItemA1['_id'])]}),
'metadata':
json.dumps({
'keya': 'valuea',
'keyb.keyc': 'valuec'
})
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, 1)
meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta']
self.assertEqual(meta['keya'], 'valuea')
self.assertEqual(meta['keyb']['keyc'], 'valuec')
resp = self.request(
method='PUT',
path='/resource/metadata',
user=self.admin,
params={
'resources':
json.dumps({
'item':
[str(self.colItemA1['_id']),
str(self.colItemB1['_id'])],
'folder': [str(self.colFolderA['_id'])]
}),
'metadata':
json.dumps({
'keya': 'valuea',
'keyb.keyc': None,
'keyb.keyd': 'valued',
})
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, 3)
meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta']
self.assertEqual(meta['keya'], 'valuea')
self.assertNotIn('keyc', meta['keyb'])
self.assertEqual(meta['keyb']['keyd'], 'valued')
resp = self.request(method='PUT',
path='/resource/metadata',
user=self.admin,
params={
'resources':
json.dumps({
'item': [
str(self.colItemA1['_id']),
],
'folder': [str(self.colFolderA['_id'])]
}),
'metadata':
json.dumps({
'keya': 'valuea',
'keyb.keyc': None,
'keyb.keyd': 'valued',
}),
'allowNull':
True
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, 2)
meta = Item().load(self.colItemA1['_id'], user=self.admin)['meta']
self.assertEqual(meta['keya'], 'valuea')
self.assertIsNone(meta['keyb']['keyc'])
self.assertEqual(meta['keyb']['keyd'], 'valued')
def initApp(self, name, app_description, collection_id, collection_name,
collection_description, public):
"""
Create the directory for start a new application. By default, without specifying
a ``collection_id``, it will create a new collection name either ``collection_name``
if provided, or **'Applications'**. If the collection 'Applications already exist it will
get it.
Return the new application (as a folder) that always contain a default
sub-folder named 'nightly'.
:param name: Name of the new application
:param app_description: Description of the new application
:param collection_id: Id of the collection within create the application
:param collection_name: Name of the collection which will be created
:param collection_description: Description of the new collection
:param public: Whether the new collection should be publicly visible
:return: The new application folder
"""
creator = self.getCurrentUser()
# Load or create the collection that contain the application
if collection_id:
collection = Collection().load(collection_id, force=True)
elif collection_name:
collection = list(Collection().find({'name': collection_name},
user=creator))
if not collection:
collection = Collection().createCollection(
name=collection_name,
description=collection_description,
public=public,
creator=creator)
else:
collection = collection[0]
else:
collection = list(Collection().find({'name': 'Applications'},
user=creator))
if not collection:
collection = Collection().createCollection(
name='Applications',
description=collection_description,
public=public,
creator=creator)
else:
collection = collection[0]
# Create the application
if not app_description:
app_description = ''
app = self._model.createFolder(parent=collection,
name=name,
description=app_description,
parentType='Collection',
public=public,
creator=creator)
# Create the 'nightly' release which will be the default folder when uploading an extension
self._model.createFolder(
parent=app,
name=constants.NIGHTLY_RELEASE_NAME,
description='Uploaded each night, always up-to-date',
parentType='Folder',
public=public,
creator=creator)
# Set a default template name for extensions in the application,
# this can be changed in anytime.
return self._model.setMetadata(
app, {
'extensionNameTemplate':
'{app_revision}_{os}_{arch}_{baseName}_{revision}'
})
def reviewImages(self, dataset, acceptedImages, flaggedImages, user):
# Avoid circular import
from .image import Image
# Verify that all images are pending review
prereviewFolder = self.prereviewFolder(dataset)
if not prereviewFolder:
raise GirderException('No Pre-review folder for this dataset.')
for image in itertools.chain(acceptedImages, flaggedImages):
if image['folderId'] != prereviewFolder['_id']:
raise ValidationException('Image %s is not in Pre-review.' %
image['_id'])
now = datetime.datetime.utcnow()
for image in acceptedImages:
image = Image().setMetadata(image, {
'reviewed': {
'userId': user['_id'],
'time': now,
'accepted': True
}
})
Image().move(image, dataset)
if flaggedImages:
flaggedCollection = Collection().findOne(
{'name': 'Flagged Images'})
flaggedFolder = Folder().findOne({
'name':
dataset['name'],
'parentId':
flaggedCollection['_id'],
'parentCollection':
'collection'
})
if not flaggedFolder:
flaggedFolder = Folder().createFolder(parent=flaggedCollection,
name=dataset['name'],
parentType='collection',
public=None,
creator=user,
allowRename=False,
reuseExisting=False)
flaggedFolder = Folder().copyAccessPolicies(prereviewFolder,
flaggedFolder,
save=True)
for image in flaggedImages:
image = Image().setMetadata(
image, {
'reviewed': {
'userId': user['_id'],
'time': now,
'accepted': False
}
})
Image().move(image, flaggedFolder)
# Remove an empty Pre-review folder
if (Folder().countItems(prereviewFolder) +
Folder().countFolders(prereviewFolder)) == 0:
Folder().remove(prereviewFolder)
def testSftpService(self):
users = ({
'email': '*****@*****.**',
'login': '******',
'firstName': 'First',
'lastName': 'Last',
'password': '******'
}, {
'email': '*****@*****.**',
'login': '******',
'firstName': 'First',
'lastName': 'Last',
'password': '******'
})
admin, user = [User().createUser(**user) for user in users]
collections = ({
'name': 'public collection',
'public': True,
'creator': admin
}, {
'name': 'private collection',
'public': False,
'creator': admin
})
privateFolder = Folder().findOne({
'parentCollection': 'user',
'parentId': user['_id'],
'name': 'Private'
})
self.assertIsNotNone(privateFolder)
Upload().uploadFromFile(six.BytesIO(b'hello world'),
size=11,
name='test.txt',
parentType='folder',
parent=privateFolder,
user=user)
for coll in collections:
Collection().createCollection(**coll)
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Incorrect password should raise authentication error
with self.assertRaises(paramiko.AuthenticationException):
client.connect('localhost',
TEST_PORT,
username='******',
password='******',
look_for_keys=False,
allow_agent=False)
# Authenticate as admin
client.connect('localhost',
TEST_PORT,
username='******',
password='******',
look_for_keys=False,
allow_agent=False)
sftpClient = client.open_sftp()
self.assertEqual(sftpClient.listdir('/'), ['collection', 'user'])
# Listing an invalid top level entity should fail
with self.assertRaises(IOError):
sftpClient.listdir('/foo')
# Test listing of users, collections, and subfolders
self.assertEqual(set(sftpClient.listdir('/user/')),
{'admin', 'regularuser'})
self.assertEqual(set(sftpClient.listdir('/user/admin')),
{'Public', 'Private'})
self.assertEqual(set(sftpClient.listdir('/collection')),
{'public collection', 'private collection'})
self.assertEqual(sftpClient.listdir('/user/regularuser/Private'),
['test.txt'])
self.assertEqual(
sftpClient.listdir('/user/regularuser/Private/test.txt'),
['test.txt'])
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.listdir('/user/nonexistent')
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.file('/user/regularuser/Private')
# Read a file using small enough buf size to require multiple chunks internally.
file = sftpClient.file('/user/regularuser/Private/test.txt/test.txt',
'r',
bufsize=4)
self.assertEqual(file.read(2), b'he')
self.assertEqual(file.read(), b'llo world')
# Make sure we enforce max buffer length
tmp, sftp.MAX_BUF_LEN = sftp.MAX_BUF_LEN, 2
file = sftpClient.file('/user/regularuser/Private/test.txt/test.txt',
'r',
bufsize=4)
with self.assertRaises(IOError):
file.read()
sftp.MAX_BUF_LEN = tmp
# Test stat capability
info = sftpClient.stat('/user/regularuser/Private')
self.assertTrue(stat.S_ISDIR(info.st_mode))
self.assertFalse(stat.S_ISREG(info.st_mode))
self.assertEqual(info.st_mode & 0o777, 0o777)
# lstat should also work
info = sftpClient.lstat('/user/regularuser/Private/test.txt/test.txt')
self.assertFalse(stat.S_ISDIR(info.st_mode))
self.assertTrue(stat.S_ISREG(info.st_mode))
self.assertEqual(info.st_size, 11)
self.assertEqual(info.st_mode & 0o777, 0o777)
# File stat implementations should agree
info = file.stat()
self.assertFalse(stat.S_ISDIR(info.st_mode))
self.assertTrue(stat.S_ISREG(info.st_mode))
self.assertEqual(info.st_size, 11)
self.assertEqual(info.st_mode & 0o777, 0o777)
# Make sure we can stat the top-level entities
for path in ('/', '/user', '/collection'):
info = sftpClient.stat(path)
self.assertTrue(stat.S_ISDIR(info.st_mode))
self.assertFalse(stat.S_ISREG(info.st_mode))
self.assertEqual(info.st_mode & 0o777, 0o777)
sftpClient.close()
client.close()
# Test that any username other than anonymous will fail using auth_none.
sock = socket.socket()
sock.connect(('localhost', TEST_PORT))
trans = paramiko.Transport(sock)
trans.connect()
with self.assertRaises(paramiko.ssh_exception.BadAuthenticationType):
trans.auth_none('')
trans.close()
sock.close()
sock = socket.socket()
sock.connect(('localhost', TEST_PORT))
trans = paramiko.Transport(sock)
trans.connect()
with self.assertRaises(paramiko.ssh_exception.BadAuthenticationType):
trans.auth_none('eponymous')
trans.close()
sock.close()
# Test that a connection can be opened for anonymous access using auth_none.
sock = socket.socket()
sock.connect(('localhost', TEST_PORT))
trans = paramiko.Transport(sock)
trans.connect()
trans.auth_none(username='******')
sftpClient = paramiko.SFTPClient.from_transport(trans)
# Only public data should be visible
self.assertEqual(set(sftpClient.listdir('/user')),
{'admin', 'regularuser'})
self.assertEqual(sftpClient.listdir('/collection'),
['public collection'])
self.assertEqual(sftpClient.listdir('/user/admin'), ['Public'])
# Make sure the client cannot distinguish between a resource that does not exist
# vs. one they simply don't have read access to.
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.listdir('/user/regularuser/Private')
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r')
sftpClient.close()
trans.close()
sock.close()
# Test anonymous access
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect('localhost',
TEST_PORT,
username='******',
password='',
look_for_keys=False,
allow_agent=False)
sftpClient = client.open_sftp()
# Only public data should be visible
self.assertEqual(set(sftpClient.listdir('/user')),
{'admin', 'regularuser'})
self.assertEqual(sftpClient.listdir('/collection'),
['public collection'])
self.assertEqual(sftpClient.listdir('/user/admin'), ['Public'])
# Make sure the client cannot distinguish between a resource that does not exist
# vs. one they simply don't have read access to.
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.listdir('/user/regularuser/Private')
with six.assertRaisesRegex(self, IOError, 'No such file'):
sftpClient.file('/user/regularuser/Private/test.txt/test.txt', 'r')
sftpClient.close()
client.close()
def stats(self, params):
drafts = get_or_create_drafts_collection()
draft_count = Collection().countFolders(drafts)
# TODO no way to publish
published_count = 0
user_count = User().collection.count_documents({})
# These aggregate queries are not indexable.
# Here are some benchmarks I generated for this endpoint as is:
# 0 dandisets | 0.008207440376281738 seconds
# 1000 dandisets | 0.019745850563049318 seconds
# 2000 dandisets | 0.030080437660217285 seconds
# 1000 more dandisets takes about .011 more seconds
# Here is the same benchmark but with these aggregate queries removed:
# 0 dandisets | 0.004081225395202637
# 1000 dandisets | 0.008220505714416505
# 2000 dandisets | 0.011307811737060547
# 1000 more dandisets takes about 0.0035 more seconds
species_count = list(Folder().collection.aggregate([
{
"$unwind": "$meta.dandiset.organism"
},
{
"$group": {
"_id": "$meta.dandiset.organism.species"
}
},
{
"$group": {
"_id": "0",
"count": {
"$sum": 1
}
}
},
]))
if species_count:
species_count = species_count[0]["count"]
else:
species_count = 0
subject_count = list(Folder().collection.aggregate([{
"$group": {
"_id": "0",
"count": {
"$sum": "$meta.dandiset.number_of_subjects"
}
}
}]))
if subject_count:
subject_count = subject_count[0]["count"]
else:
subject_count = 0
cell_count = list(Folder().collection.aggregate([{
"$group": {
"_id": "0",
"count": {
"$sum": "$meta.dandiset.number_of_cells"
}
}
}]))
if cell_count:
cell_count = cell_count[0]["count"]
else:
cell_count = 0
return {
"draft_count": draft_count,
"published_count": published_count,
"user_count": user_count,
"species_count": species_count,
"subject_count": subject_count,
"cell_count": cell_count,
"size": drafts["size"],
}
# If there is are no users, create an admin user
if User().findOne() is None:
User().createUser('admin', 'password', 'Admin', 'Admin',
'*****@*****.**')
adminUser = User().findOne({'admin': True})
# Configure asset stores
if Assetstore().findOne() is None:
Assetstore().createFilesystemAssetstore('Assetstore', '/assetstore')
Assetstore().createFilesystemAssetstore('SMM', '/Datashare/SMM')
Assetstore().createFilesystemAssetstore('GlusterFS', '/glusterfs/SMM')
# If we don't have a default task folder, make a task collection and folder
if not Setting().get('slicer_cli_web.task_folder'):
# Make sure we have a Tasks collection with a Slicer CLI Web Tasks folder
if Collection().findOne({'name': 'Tasks'}) is None:
Collection().createCollection('Tasks', adminUser)
tasksCollection = Collection().findOne({'name': 'Tasks'})
taskFolderName = 'Slicer CLI Web Tasks'
if Folder().findOne({
'name': taskFolderName,
'parentId': tasksCollection['_id']
}) is None:
Folder().createFolder(tasksCollection,
taskFolderName,
parentType='collection',
public=True,
creator=adminUser)
taskFolder = Folder().findOne({
'name': taskFolderName,
'parentId': tasksCollection['_id']
def get_or_create_drafts_collection():
return Collection().createCollection(DANDI_DRAFTS_COLLECTION_NAME, reuseExisting=True)
def testYAMLConfigFile(server, admin, user, fsAssetstore):
# Create some resources to use in the tests
collection = Collection().createCollection('collection A', admin)
colFolderA = Folder().createFolder(collection,
'folder A',
parentType='collection',
creator=admin)
colFolderB = Folder().createFolder(colFolderA, 'folder B', creator=admin)
groupA = Group().createGroup('Group A', admin)
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
method='GET')
assert utilities.respStatus(resp) == 200
assert resp.json is None
colFolderConfig = Folder().createFolder(collection,
'.config',
parentType='collection',
creator=admin)
utilities.uploadText(json.dumps({'keyA': 'value1'}), admin, fsAssetstore,
colFolderConfig, 'sample.json')
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']))
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value1'
utilities.uploadText(json.dumps({'keyA': 'value2'}), admin, fsAssetstore,
colFolderA, 'sample.json')
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']))
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value2'
utilities.uploadText(
json.dumps({
'keyA': 'value3',
'groups': {
'Group A': {
'access': {
'user': {
'keyA': 'value4'
}
}
}
},
'access': {
'user': {
'keyA': 'value5'
},
'admin': {
'keyA': 'value6'
}
}
}), admin, fsAssetstore, colFolderB, 'sample.json')
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']))
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value3'
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=user)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value5'
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value6'
Group().addUser(groupA, user)
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=user)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value4'
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value6'
def _testDownloadCollection(self):
"""
Test downloading an entire collection as a zip file.
"""
# Create a collection
resp = self.request(
path='/collection', method='POST', user=self.user, params={
'name': 'Test Collection'
})
self.assertStatusOk(resp)
collection = resp.json
# Create a folder in the collection
resp = self.request(
path='/folder', method='POST', user=self.user, params={
'name': 'Test Folder',
'parentId': collection['_id'],
'parentType': 'collection'
})
self.assertStatusOk(resp)
test = resp.json
contents = os.urandom(64) # Generate random file contents
# Upload the file into that subfolder
resp = self.request(
path='/file', method='POST', user=self.user, params={
'parentType': 'folder',
'parentId': test['_id'],
'name': 'random.bin',
'size': len(contents)
})
self.assertStatusOk(resp)
uploadId = resp.json['_id']
# Send the file contents
resp = self.request(
path='/file/chunk', method='POST', user=self.user, body=contents, params={
'uploadId': uploadId
}, type='application/octet-stream')
self.assertStatusOk(resp)
# Download the collection
path = '/collection/%s/download' % collection['_id']
resp = self.request(
path=path,
method='GET', user=self.user, isJson=False)
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Disposition'],
'attachment; filename="Test Collection.zip"')
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
extracted = zip.read('Test Collection/Test Folder/random.bin')
self.assertEqual(extracted, contents)
# Make collection public
collection = Collection().load(collection['_id'], force=True)
collection['public'] = True
collection = Collection().save(collection)
# Download the collection as anonymous
path = '/collection/%s/download' % str(collection['_id'])
resp = self.request(
path=path,
method='GET', user=None, isJson=False)
self.assertStatusOk(resp)
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
# Zip file should have no entries
self.assertFalse(zip.namelist())
# Upload a known MIME-type file into the collection
contents = b'not a jpeg'
resp = self.request(
path='/file', method='POST', user=self.user, params={
'parentType': 'folder',
'parentId': test['_id'],
'name': 'fake.jpeg',
'size': len(contents),
'mimeType': 'image/jpeg'
})
self.assertStatusOk(resp)
uploadId = resp.json['_id']
# Send the file contents
resp = self.request(
path='/file/chunk', method='POST', user=self.user, body=contents, params={
'uploadId': uploadId
}, type='application/octet-stream')
self.assertStatusOk(resp)
# Download the collection using a MIME type filter
path = '/collection/%s/download' % str(collection['_id'])
resp = self.request(
path=path, method='GET', user=self.user, isJson=False, params={
'mimeFilter': json.dumps(['image/png', 'image/jpeg'])
})
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Disposition'],
'attachment; filename="Test Collection.zip"')
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
# Only the jpeg should exist in the zip
path = 'Test Collection/Test Folder/fake.jpeg'
self.assertEqual(zip.namelist(), [path])
extracted = zip.read(path)
self.assertEqual(extracted, contents)
def testYAMLConfigFileInherit(server, admin, user, fsAssetstore):
# Create some resources to use in the tests
collection = Collection().createCollection('collection A', admin)
colFolderA = Folder().createFolder(collection,
'folder A',
parentType='collection',
creator=admin)
colFolderB = Folder().createFolder(colFolderA, 'folder B', creator=admin)
colFolderConfig = Folder().createFolder(collection,
'.config',
parentType='collection',
creator=admin)
collectionB = Collection().createCollection('collection B', admin)
configFolder = Folder().createFolder(collectionB,
'any',
parentType='collection',
creator=admin)
Setting().set(constants.PluginSettings.LARGE_IMAGE_CONFIG_FOLDER,
str(configFolder['_id']))
utilities.uploadText(
json.dumps({
'keyA': 'value1',
'keyB': 'value2',
'keyC': 'value3',
'__inherit__': True
}), admin, fsAssetstore, colFolderB, 'sample.json')
utilities.uploadText(
json.dumps({
'keyA': 'value4',
'keyD': 'value5',
'__inherit__': True
}), admin, fsAssetstore, colFolderConfig, 'sample.json')
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value1'
assert resp.json['keyB'] == 'value2'
assert resp.json['keyC'] == 'value3'
assert resp.json['keyD'] == 'value5'
utilities.uploadText(json.dumps({
'keyB': 'value6',
'keyE': 'value7'
}), admin, fsAssetstore, configFolder, 'sample.json')
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value1'
assert resp.json['keyB'] == 'value2'
assert resp.json['keyC'] == 'value3'
assert resp.json['keyD'] == 'value5'
assert resp.json['keyE'] == 'value7'
Folder().remove(colFolderConfig, user=admin)
resp = server.request(path='/folder/%s/yaml_config/sample.json' %
str(colFolderB['_id']),
user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json['keyA'] == 'value1'
assert resp.json['keyB'] == 'value2'
assert resp.json['keyC'] == 'value3'
assert resp.json['keyE'] == 'value7'
def testResourceSearch(self):
"""
Test resource/search endpoint
"""
# get expected models from the database
admin = User().findOne({'login': '******'})
user = User().findOne({'login': '******'})
coll1 = Collection().findOne({'name': 'Test Collection'})
coll2 = Collection().findOne({'name': 'Magic collection'})
item1 = Item().findOne({'name': 'Public object'})
# set user read permissions on the private collection
Collection().setUserAccess(coll2,
user,
level=AccessType.READ,
save=True)
# Grab the default user folders
resp = self.request(path='/folder',
method='GET',
user=user,
params={
'parentType': 'user',
'parentId': user['_id'],
'sort': 'name',
'sortdir': 1
})
privateFolder = resp.json[0]
# First test all of the required parameters.
self.ensureRequiredParams(path='/resource/search',
required=['q', 'types'])
# Now test parameter validation
resp = self.request(path='/resource/search',
params={
'q': 'query',
'types': ',,invalid;json!'
})
self.assertStatus(resp, 400)
self.assertEqual('Parameter types must be valid JSON.',
resp.json['message'])
# Test searching with no results
resp = self.request(path='/resource/search',
params={
'q':
'gibberish',
'types':
'["folder", "user", "collection", "group"]'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, {
'folder': [],
'user': [],
'collection': [],
'group': []
})
# Ensure searching respects permissions
resp = self.request(path='/resource/search',
params={
'q': 'private',
'types': '["folder", "user", "collection"]'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, {
'folder': [],
'user': [],
'collection': []
})
resp = self.request(path='/resource/search',
params={
'q': 'pr',
'mode': 'prefix',
'types': '["folder", "user", "collection"]'
})
self.assertStatusOk(resp)
self.assertEqual(resp.json, {
'folder': [],
'user': [],
'collection': []
})
resp = self.request(path='/resource/search',
params={
'q': 'private',
'types': '["folder", "user", "collection"]'
},
user=user)
self.assertStatusOk(resp)
self.assertEqual(1, len(resp.json['folder']))
self.assertDictContainsSubset(
{
'_id': str(privateFolder['_id']),
'name': 'Private'
}, resp.json['folder'][0])
self.assertEqual(1, len(resp.json['collection']))
self.assertDictContainsSubset(
{
'_id': str(coll2['_id']),
'name': coll2['name']
}, resp.json['collection'][0])
self.assertEqual(0, len(resp.json['user']))
resp = self.request(path='/resource/search',
params={
'q': 'pr',
'mode': 'prefix',
'types':
'["folder", "user", "collection", "item"]'
},
user=user)
self.assertStatusOk(resp)
self.assertEqual(1, len(resp.json['folder']))
self.assertDictContainsSubset(
{
'_id': str(privateFolder['_id']),
'name': 'Private'
}, resp.json['folder'][0])
self.assertEqual(0, len(resp.json['collection']))
self.assertEqual(0, len(resp.json['item']))
self.assertEqual(0, len(resp.json['user']))
# Ensure that weights are respected, e.g. description should be
# weighted less than name.
resp = self.request(path='/resource/search',
params={
'q': 'magic',
'types': '["collection"]'
},
user=admin)
self.assertStatusOk(resp)
self.assertEqual(2, len(resp.json['collection']))
self.assertDictContainsSubset(
{
'_id': str(coll2['_id']),
'name': coll2['name']
}, resp.json['collection'][0])
self.assertDictContainsSubset(
{
'_id': str(coll1['_id']),
'name': coll1['name']
}, resp.json['collection'][1])
self.assertTrue(resp.json['collection'][0]['_textScore'] >
resp.json['collection'][1]['_textScore'])
# Exercise user search by login
resp = self.request(path='/resource/search',
params={
'q': 'goodlogin',
'types': '["user"]'
},
user=admin)
self.assertStatusOk(resp)
self.assertEqual(1, len(resp.json['user']))
self.assertDictContainsSubset(
{
'_id': str(user['_id']),
'firstName': user['firstName'],
'lastName': user['lastName'],
'login': user['login']
}, resp.json['user'][0])
# check item search with proper permissions
resp = self.request(path='/resource/search',
params={
'q': 'object',
'types': '["item"]'
},
user=user)
self.assertStatusOk(resp)
self.assertEqual(1, len(resp.json['item']))
self.assertDictContainsSubset(
{
'_id': str(item1['_id']),
'name': item1['name']
}, resp.json['item'][0])
# Check search for model that is not access controlled
self.assertNotIsInstance(Assetstore(), AccessControlledModel)
self.assertNotIsInstance(Assetstore(), AccessControlMixin)
resource.allowedSearchTypes.add('assetstore')
resp = self.request(path='/resource/search',
params={
'q': 'Test',
'mode': 'prefix',
'types': '["assetstore"]'
},
user=user)
self.assertStatusOk(resp)
self.assertEqual(1, len(resp.json['assetstore']))
def setUp(self):
base.TestCase.setUp(self)
# Create a user and a data hierarchy
admin = {
'email': '*****@*****.**',
'login': '******',
'firstName': 'Admin',
'lastName': 'Last',
'password': '******',
'admin': True
}
self.admin = User().createUser(**admin)
coll1 = {'name': 'Coll1', 'creator': self.admin}
self.coll1 = Collection().createCollection(**coll1)
coll2 = {'name': 'Coll2', 'creator': self.admin}
self.coll2 = Collection().createCollection(**coll2)
folder1 = {
'parent': self.coll1,
'creator': self.admin,
'parentType': 'collection',
'name': 'Top level folder'
}
self.folder1 = Folder().createFolder(**folder1)
folder2 = {
'parent': self.folder1,
'creator': self.admin,
'parentType': 'folder',
'name': 'Subfolder'
}
self.folder2 = Folder().createFolder(**folder2)
item1 = {
'name': 'Item1',
'creator': self.admin,
'folder': self.folder1
}
self.item1 = Item().createItem(**item1)
item10 = {
'name': 'Item10',
'creator': self.admin,
'folder': self.folder2
}
self.item10 = Item().createItem(**item10)
file1 = {
'creator': self.admin,
'item': self.item1,
'name': 'File1',
'size': 1,
'assetstore': self.assetstore,
'mimeType': 'text/plain'
}
self.file1 = File().createFile(**file1)
self.file1['sha512'] = ''
File().save(self.file1, validate=False)
file10 = {
'creator': self.admin,
'item': self.item10,
'name': 'File10',
'size': 10,
'assetstore': self.assetstore,
'mimeType': 'text/plain'
}
self.file10 = File().createFile(**file10)
self.file10['sha512'] = ''
File().save(self.file10, validate=False)
def collections(db):
yield (Collection().createCollection('private collection', public=False),
Collection().createCollection('public collection', public=True))
def testCollectionAccess(self):
# Asking to change to an invalid access list should fail
resp = self.request(path='/collection/%s/access' %
self.collection['_id'],
method='PUT',
params={
'access': 'not an access list',
'public': False
},
user=self.admin)
self.assertStatus(resp, 400)
# Create some folders underneath the collection
folder1 = Folder().createFolder(parentType='collection',
parent=self.collection,
creator=self.admin,
public=False,
name='top level')
folder2 = Folder().createFolder(parentType='folder',
parent=folder1,
creator=self.admin,
public=False,
name='subfolder')
Folder().createFolder(parentType='collection',
parent=self.collection,
creator=self.admin,
public=False,
name='another top level folder')
# Admin should see two top level folders
resp = self.request(path='/collection/%s/details' %
self.collection['_id'],
user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json['nFolders'], 2)
self.assertNotIn('nItems', resp.json)
# Normal user should see 0 folders
resp = self.request(path='/collection/%s/details' %
self.collection['_id'],
user=self.user)
self.assertStatusOk(resp)
self.assertEqual(resp.json['nFolders'], 0)
# Add read access on one of the folders
Folder().setUserAccess(folder1, self.user, AccessType.READ, save=True)
# Normal user should see one folder now
resp = self.request(path='/collection/%s/details' %
self.collection['_id'],
user=self.user)
self.assertStatusOk(resp)
self.assertEqual(resp.json['nFolders'], 1)
# Change the access to allow just the user
obj = {
'users': [{
'id': str(self.user['_id']),
'level': AccessType.WRITE
}]
}
resp = self.request(path='/collection/%s/access' %
self.collection['_id'],
method='PUT',
params={
'access': json.dumps(obj),
'public': True
},
user=self.admin)
self.assertStatusOk(resp)
# Request the collection access
resp = self.request(path='/collection/%s/access' %
self.collection['_id'],
user=self.admin)
self.assertStatusOk(resp)
access = resp.json
self.assertEqual(access['users'][0]['id'], str(self.user['_id']))
self.assertEqual(access['users'][0]['level'], AccessType.WRITE)
coll = Collection().load(self.collection['_id'], force=True)
folder1 = Folder().load(folder1['_id'], force=True)
folder2 = Folder().load(folder2['_id'], force=True)
self.assertEqual(coll['public'], True)
self.assertEqual(folder1['public'], False)
# Update the collection recursively to public
resp = self.request(path='/collection/%s/access' % coll['_id'],
method='PUT',
params={
'access': json.dumps(obj),
'public': True,
'recurse': True,
'progress': True
},
user=self.admin)
self.assertStatusOk(resp)
coll = Collection().load(coll['_id'], force=True)
folder1 = Folder().load(folder1['_id'], force=True)
folder2 = Folder().load(folder2['_id'], force=True)
self.assertEqual(coll['public'], True)
self.assertEqual(folder1['public'], True)
self.assertEqual(folder2['public'], True)
self.assertEqual(folder1['access'], coll['access'])
self.assertEqual(folder1['access'], folder2['access'])
self.assertEqual(
folder2['access'], {
'users': [{
'id': self.user['_id'],
'level': AccessType.WRITE,
'flags': []
}],
'groups': []
})
# Recursively drop the user's access level to READ
obj['users'][0]['level'] = AccessType.READ
resp = self.request(path='/collection/%s/access' % coll['_id'],
method='PUT',
params={
'access': json.dumps(obj),
'public': True,
'recurse': True,
'progress': True
},
user=self.admin)
self.assertStatusOk(resp)
coll = Collection().load(coll['_id'], force=True)
folder1 = Folder().load(folder1['_id'], force=True)
folder2 = Folder().load(folder2['_id'], force=True)
self.assertEqual(coll['public'], True)
self.assertEqual(folder1['public'], True)
self.assertEqual(folder2['public'], True)
self.assertEqual(folder1['access'], coll['access'])
self.assertEqual(folder1['access'], folder2['access'])
self.assertEqual(
folder2['access'], {
'users': [{
'id': self.user['_id'],
'level': AccessType.READ,
'flags': []
}],
'groups': []
})
# Recursively remove the user's access altogether, also make sure that
# passing no "public" param just retains the current flag state
obj['users'] = ()
resp = self.request(path='/collection/%s/access' % coll['_id'],
method='PUT',
params={
'access': json.dumps(obj),
'recurse': True
},
user=self.admin)
self.assertStatusOk(resp)
coll = Collection().load(coll['_id'], force=True)
folder1 = Folder().load(folder1['_id'], force=True)
folder2 = Folder().load(folder2['_id'], force=True)
self.assertEqual(coll['public'], True)
self.assertEqual(folder1['public'], True)
self.assertEqual(folder2['public'], True)
self.assertEqual(folder1['access'], coll['access'])
self.assertEqual(folder1['access'], folder2['access'])
self.assertEqual(folder2['access'], {'users': [], 'groups': []})
# Add group access to the collection
group = Group().createGroup('test', self.admin)
obj = {
'groups': [{
'id': str(group['_id']),
'level': AccessType.WRITE
}]
}
resp = self.request(path='/collection/%s/access' % coll['_id'],
method='PUT',
params={
'access': json.dumps(obj),
'recurse': False
},
user=self.admin)
self.assertStatusOk(resp)
# Create a new top-level folder, it should inherit the collection ACL.
resp = self.request(path='/folder',
method='POST',
params={
'name': 'top level 2',
'parentId': coll['_id'],
'parentType': 'collection'
},
user=self.admin)
self.assertStatusOk(resp)
folder = Folder().load(resp.json['_id'], force=True)
coll = Collection().load(coll['_id'], force=True)
self.assertEqual(coll['access']['users'], [])
self.assertEqual(folder['access']['users'], [{
'id': self.admin['_id'],
'level': AccessType.ADMIN,
'flags': []
}])
self.assertEqual(folder['access']['groups'], [{
'id': group['_id'],
'level': AccessType.WRITE,
'flags': []
}])
self.assertEqual(folder['access']['groups'], coll['access']['groups'])
def testAccessFlags(self):
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(resp.json, {})
registerAccessFlag('my_key', name='hello', description='a custom flag')
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(
resp.json, {
'my_key': {
'name': 'hello',
'description': 'a custom flag',
'admin': False
}
})
self.users[1] = User().load(self.users[1]['_id'], force=True)
user = self.users[1]
# Manage custom access flags on an access controlled resource
self.assertFalse(User().hasAccessFlags(user, user, flags=['my_key']))
# Admin should always have permission
self.assertTrue(User().hasAccessFlags(user,
self.users[0],
flags=['my_key']))
# Test the requireAccessFlags method
with self.assertRaises(AccessException):
User().requireAccessFlags(user, user=user, flags='my_key')
User().requireAccessFlags(user, user=self.users[0], flags='my_key')
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Test loadmodel requiredFlags argument via REST endpoint
resp = self.request('/test_endpoints/loadmodel_with_flags/%s' %
user['_id'],
user=self.users[1])
self.assertStatus(resp, 403)
user = User().setAccessList(
self.users[0],
access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'not a registered flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key']
}]
},
save=True)
resp = self.request('/test_endpoints/loadmodel_with_flags/%s' %
user['_id'],
user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(resp.json, 'success')
# Only registered flags should be stored
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['my_key'])
self.assertTrue(User().hasAccessFlags(user, user, flags=['my_key']))
# Create an admin-only access flag
registerAccessFlag('admin_flag', name='admin flag', admin=True)
# Non-admin shouldn't be able to set it
user = User().setAccessList(self.users[0],
access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': []
},
save=True,
user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Admin user should be able to set it
user = User().setAccessList(self.users[1],
access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
},
save=True,
user=self.users[0])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['admin_flag'])
# An already-enabled admin-only flag should stay enabled for non-admin user
user = User().setAccessList(self.users[1],
access={
'users': [{
'id':
self.users[1]['_id'],
'level':
AccessType.ADMIN,
'flags': ['my_key', 'admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
},
save=True,
user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(set(acl['users'][0]['flags']),
{'my_key', 'admin_flag'})
self.assertEqual(acl['groups'][0]['flags'], ['admin_flag'])
# Test setting public flags on a collection and folder
collectionModel = Collection()
folderModel = Folder()
itemModel = Item()
collection = collectionModel.createCollection('coll',
creator=self.users[0],
public=True)
folder = folderModel.createFolder(collection,
'folder',
parentType='collection',
creator=self.users[0])
# Add an item to the folder so we can test AclMixin flag behavior
item = itemModel.createItem(folder=folder,
name='test',
creator=self.users[0])
folder = folderModel.setUserAccess(folder,
self.users[1],
level=AccessType.ADMIN,
save=True,
currentUser=self.users[0])
with self.assertRaises(AccessException):
collectionModel.requireAccessFlags(collection,
user=None,
flags='my_key')
# Test AclMixin flag behavior
with self.assertRaises(AccessException):
itemModel.requireAccessFlags(item, user=None, flags='my_key')
self.assertFalse(
itemModel.hasAccessFlags(item, user=None, flags='my_key'))
collection = collectionModel.setAccessList(collection,
access=collection['access'],
save=True,
recurse=True,
user=self.users[0],
publicFlags=['my_key'])
collectionModel.requireAccessFlags(collection,
user=None,
flags='my_key')
# Make sure recursive setting of public flags worked
folder = folderModel.load(folder['_id'], force=True)
self.assertEqual(folder['publicFlags'], ['my_key'])
itemModel.requireAccessFlags(item, user=None, flags='my_key')
# Non-admin shouldn't be able to set admin-only public flags
folder = folderModel.setPublicFlags(folder,
flags=['admin_flag'],
user=self.users[1],
save=True)
self.assertEqual(folder['publicFlags'], [])
# Admin users should be able to set admin-only public flags
folder = folderModel.setPublicFlags(folder,
flags=['admin_flag'],
user=self.users[0],
save=True,
append=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
# Non-admin users can set admin-only public flags if they are already enabled
folder = folderModel.setPublicFlags(folder,
flags=['admin_flag', 'my_key'],
user=self.users[1],
save=True)
self.assertEqual(set(folder['publicFlags']), {'admin_flag', 'my_key'})
# Test "force" options
folder = folderModel.setPublicFlags(folder,
flags='admin_flag',
force=True,
save=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
folder = folderModel.setAccessList(folder,
access={
'users': [{
'id':
self.users[1]['_id'],
'level':
AccessType.ADMIN,
'flags':
['my_key', 'admin_flag']
}],
'groups': []
},
save=True,
force=True)
folderModel.requireAccessFlags(folder,
user=self.users[1],
flags='my_key')
folder = folderModel.setUserAccess(folder,
self.users[1],
level=AccessType.READ,
save=True,
force=True,
flags=[])
self.assertFalse(
folderModel.hasAccessFlags(folder, self.users[1], flags='my_key'))
folder = folderModel.setGroupAccess(folder,
self.group,
level=AccessType.READ,
save=True,
force=True,
flags='my_key')
folderModel.requireAccessFlags(folder,
user=self.users[1],
flags='my_key')
# Testing with flags=None should give sensible behavior
folderModel.requireAccessFlags(folder, user=None, flags=None)
# Test filtering results by access flags (both ACModel and AclMixin)
for model, doc in ((folderModel, folder), (itemModel, item)):
cursor = model.find({})
self.assertGreater(len(list(cursor)), 0)
cursor = model.find({})
filtered = list(
model.filterResultsByPermission(cursor,
user=None,
level=AccessType.READ,
flags='my_key'))
self.assertEqual(len(filtered), 0)
cursor = model.find({})
filtered = list(
model.filterResultsByPermission(cursor,
user=self.users[1],
level=AccessType.READ,
flags=('my_key',
'admin_flag')))
self.assertEqual(len(filtered), 1)
self.assertEqual(filtered[0]['_id'], doc['_id'])
def testEmptyCreator(self):
c = Collection().createCollection('No Creator')
self.assertEqual(c['creatorId'], None)
def _createFiles(self, user=None):
"""
Create a set of items, folders, files, metadata, and collections for
testing.
:param user: the user who should own these items.
"""
if user is None:
user = self.admin
self.expectedZip = {}
# Create a collection
coll = {
'name': 'Test Collection',
'description': 'The description',
'public': True,
'creator': user
}
self.collection = Collection().createCollection(**coll)
self.collectionPrivateFolder = Folder().createFolder(
parent=self.collection,
parentType='collection',
name='Private',
creator=user,
public=False)
# Get the admin user's folders
resp = self.request(path='/folder',
method='GET',
user=user,
params={
'parentType': 'user',
'parentId': user['_id'],
'sort': 'name',
'sortdir': 1
})
self.adminPrivateFolder = Folder().load(resp.json[0]['_id'], user=user)
self.adminPublicFolder = Folder().load(resp.json[1]['_id'], user=user)
# Create a folder within the admin public folder
resp = self.request(path='/folder',
method='POST',
user=user,
params={
'name': 'Folder 1',
'parentId': self.adminPublicFolder['_id']
})
self.adminSubFolder = resp.json
# Create a series of items
self.items = []
self.items.append(Item().createItem('Item 1', self.admin,
self.adminPublicFolder))
self.items.append(Item().createItem('Item 2', self.admin,
self.adminPublicFolder))
self.items.append(Item().createItem('It\\em/3', self.admin,
self.adminSubFolder))
self.items.append(Item().createItem('Item 4', self.admin,
self.collectionPrivateFolder))
self.items.append(Item().createItem('Item 5', self.admin,
self.collectionPrivateFolder))
# Upload a series of files
file, path, contents = self._uploadFile('File 1', self.items[0])
self.file1 = file
self.expectedZip[path] = contents
file, path, contents = self._uploadFile('File 2', self.items[0])
self.expectedZip[path] = contents
file, path, contents = self._uploadFile('File 3', self.items[1])
self.expectedZip[path] = contents
file, path, contents = self._uploadFile('File 4', self.items[2])
self.expectedZip[path] = contents
file, path, contents = self._uploadFile('File 5', self.items[3])
self.expectedZip[path] = contents
# place some metadata on two of the items and one of the folders
meta = {'key': 'value'}
Item().setMetadata(self.items[2], meta)
parents = Item().parentsToRoot(self.items[2], self.admin)
path = os.path.join(*([
part['object'].get('name', part['object'].get('login', ''))
for part in parents
] + [self.items[2]['name'], 'girder-item-metadata.json']))
self.expectedZip[path] = meta
meta = {'x': 'y'}
Item().setMetadata(self.items[4], meta)
parents = Item().parentsToRoot(self.items[4], self.admin)
path = os.path.join(*([
part['object'].get('name', part['object'].get('login', ''))
for part in parents
] + [self.items[4]['name'], 'girder-item-metadata.json']))
self.expectedZip[path] = meta
meta = {'key2': 'value2', 'date': datetime.datetime.utcnow()}
# mongo rounds to millisecond, so adjust our expectations
meta['date'] -= datetime.timedelta(
microseconds=meta['date'].microsecond % 1000)
Folder().setMetadata(self.adminPublicFolder, meta)
parents = Folder().parentsToRoot(self.adminPublicFolder, user=user)
path = os.path.join(*([
part['object'].get('name', part['object'].get('login', ''))
for part in parents
] + [self.adminPublicFolder['name'], 'girder-folder-metadata.json']))
self.expectedZip[path] = meta
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'):
"""
As needed, download sample data.
:param adminUser: a user to create and modify collections and folders.
:param collName: the collection name where the data will be added.
:param folderName: the folder name where the data will bed aded.
:returns: the folder where the sample data is located.
"""
if Collection().findOne({'lowerName': collName.lower()}) is None:
Collection().createCollection(collName, adminUser)
collection = Collection().findOne({'lowerName': collName.lower()})
if Folder().findOne({
'parentId': collection['_id'],
'lowerName': folderName.lower()
}) is None:
Folder().createFolder(collection,
folderName,
parentType='collection',
public=True,
creator=adminUser)
folder = Folder().findOne({
'parentId': collection['_id'],
'lowerName': folderName.lower()
})
remote = girder_client.GirderClient(
apiUrl='https://data.kitware.com/api/v1')
remoteFolder = remote.resourceLookup(
'/collection/HistomicsTK/Deployment test images')
sampleItems = []
for remoteItem in remote.listItem(remoteFolder['_id']):
item = Item().findOne({
'folderId': folder['_id'],
'name': remoteItem['name']
})
if item and len(list(Item().childFiles(item, limit=1))):
sampleItems.append(item)
continue
if not item:
item = Item().createItem(remoteItem['name'],
creator=adminUser,
folder=folder)
for remoteFile in remote.listFile(remoteItem['_id']):
with tempfile.NamedTemporaryFile() as tf:
fileName = tf.name
tf.close()
sys.stdout.write('Downloading %s' % remoteFile['name'])
sys.stdout.flush()
remote.downloadFile(remoteFile['_id'], fileName)
sys.stdout.write(' .')
sys.stdout.flush()
Upload().uploadFromFile(open(fileName, 'rb'),
os.path.getsize(fileName),
name=remoteItem['name'],
parentType='item',
parent=item,
user=adminUser)
sys.stdout.write('.\n')
sys.stdout.flush()
sampleItems.append(item)
for item in sampleItems:
if 'largeImage' not in item:
sys.stdout.write('Making large_item %s ' % item['name'])
sys.stdout.flush()
try:
ImageItem().createImageItem(item, createJob=False)
except Exception:
pass
print('done')
return folder
