def createNotebooks(event):
user = event.info
folder_model = Folder()
result = lookUpPath('user/%s/Private' % user['login'], force=True)
private_folder = result['document']
oc_folder = folder_model.createFolder(private_folder, 'oc',
parentType='folder',
creator=user,
public=True,
reuseExisting=True)
notebook_folder = folder_model.createFolder(oc_folder, 'notebooks',
parentType='folder',
creator=user,
public=True,
reuseExisting=True)
notebooks_dir = os.path.join(os.path.dirname(__file__), 'notebooks')
upload_model = Upload()
for file in glob.glob('%s/*.ipynb' % notebooks_dir):
size = os.path.getsize(file)
name = os.path.basename(file)
with open(file, 'rb') as fp:
upload_model.uploadFromFile(
fp, size=size, name=name, parentType='folder',
parent={'_id': ObjectId(notebook_folder['_id'])}, user=user,
mimeType='application/x-ipynb+json')
def testYAMLAliases(self):
folderModel = Folder()
aliasedFolders = list(folderModel.find({'name': 'Common'}, force=True))
self.assertTrue(len(aliasedFolders) == 2)
for folder in aliasedFolders:
self.assertTrue(
len(list(folderModel.childItems(folder, force=True))) == 2
)
def __init__(self):
super(HdfsAssetstoreResource, self).__init__()
self.resourceName = 'hdfs_assetstore'
self.route('PUT', (':id', 'import'), self.importData)
self.folderModel = Folder() # Save to avoid many lookups
self.itemModel = Item()
self.fileModel = File()
def stream():
z = ziputil.ZipGenerator(folder['name'])
for (path, file) in Folder().fileList(folder,
user=user,
subpath=False):
for data in z.addFile(file, path):
yield data
for data in z.addFile(gen, "output_tracks.csv"):
yield data
yield z.footer()
def test_mapping_creation(self):
new_folder = Folder().createFolder(
self.base_collection,
random_string(),
creator=self.users["sally"],
parentType="collection",
public=True,
reuseExisting=True,
)
resp = self.request(
path="/folder/{_id}".format(**new_folder),
method="PUT",
user=self.users["sally"],
params={
"fsPath": "/etc",
"isMapping": True
}, # H4ck3r detected!
)
self.assertStatus(resp, 403)
self.assertEqual(
resp.json,
{
"message": "Must be admin to setup virtual folders.",
"type": "access"
},
)
resp = self.request(
path="/folder/{_id}".format(**new_folder),
method="PUT",
user=self.users["admin"],
params={
"fsPath": "/etc",
"isMapping": True
}, # G0d
)
self.assertStatusOk(resp)
self.assertHasKeys(resp.json, ["fsPath", "isMapping"])
self.assertEqual(resp.json["fsPath"], "/etc")
self.assertEqual(resp.json["isMapping"], True)
Folder().remove(new_folder)
def _validateFolder(event):
doc = event.info
if 'isVirtual' in doc and not isinstance(doc['isVirtual'], bool):
raise ValidationException('The isVirtual field must be boolean.',
field='isVirtual')
if doc.get('isVirtual'):
# Make sure it doesn't have children
if list(Folder().childItems(doc, limit=1)):
raise ValidationException(
'Virtual folders may not contain child items.',
field='isVirtual')
if list(Folder().find(
{
'parentId': doc['_id'],
'parentCollection': 'folder'
}, limit=1)):
raise ValidationException(
'Virtual folders may not contain child folders.',
field='isVirtual')
if doc['parentCollection'] == 'folder':
parent = Folder().load(event.info['parentId'], force=True, exc=True)
if parent.get('isVirtual'):
raise ValidationException(
'You may not place folders under a virtual folder.',
field='folderId')
if 'virtualItemsQuery' in doc:
try:
json.loads(doc['virtualItemsQuery'])
except (TypeError, ValueError):
raise ValidationException(
'The virtual items query must be valid JSON.',
field='virtualItemsQuery')
if 'virtualItemsSort' in doc:
try:
json.loads(doc['virtualItemsSort'])
except (TypeError, ValueError):
raise ValidationException(
'The virtual items sort must be valid JSON.',
field='virtualItemsSort')
def _virtualFolderDetails(self, event):
folderId = event.info['id']
user = self.getCurrentUser()
folder = Folder().load(folderId, user=user, level=AccessType.READ)
if not folder or not folder.get(
'isVirtual') or 'virtualItemsQuery' not in folder:
return # Parent is not a virtual folder, proceed as normal
q = json_util.loads(folder['virtualItemsQuery'])
item = Item()
# Virtual folders can't contain subfolders
result = {
'nFolders':
0,
'nItems':
item.findWithPermissions(q, user=user, level=AccessType.READ).count()
}
event.preventDefault().addResponse(result)
def ensure_version(self, event: events.Event):
params = event.info.get("params", {})
taleId = event.info.get("id")
version_id = params.get("versionId")
if not version_id:
try:
version = self.create(taleId=taleId,
allowRename=True,
params={})
# Above obj is filtered model so we need to reload it...
version = Folder().load(version["_id"], force=True)
except RestException as exc:
if exc.code == 303:
version = Folder().load(exc.extra, force=True)
else:
raise
# We're using 'updated' field to bump the version to the top of
# Folder().list(). We're gonna update it either way later on.
Folder().updateFolder(version)
def testWebClientAnnotationSpec(boundServer, fsAssetstore, db, admin):
# Create an item in the Public folder
publicFolder = next(
folder for folder in Folder().childFolders(parent=admin, parentType='user', user=admin)
if folder['public'] is True)
Item().createItem('Empty', admin, publicFolder)
spec = os.path.join(os.path.dirname(__file__), 'web_client_specs', 'annotationSpec.js')
runWebClientTest(boundServer, spec, 15000)
def updateFolderAccess(self, phase, submissions):
"""
Synchronize access control between the phase and submission folders for
the phase. Phase admins should have read access on the submission
folders.
"""
# Get phase admin users
phaseAcl = Phase().getFullAccessList(phase)
phaseAdminUserIds = set([user['id']
for user in phaseAcl.get('users')
if user['level'] >= AccessType.WRITE])
phaseAdminUsers = [User().load(userId, force=True, exc=True)
for userId in phaseAdminUserIds]
# Update submission folder ACL for current phase admins
try:
for sub in submissions:
folder = Folder().load(sub['folderId'], force=True)
if not folder:
continue
folderAcl = Folder().getFullAccessList(folder)
# Revoke access to users who are not phase admins; ignore folder
# owner
usersToRemove = [User().load(user['id'], force=True,
exc=True)
for user in folderAcl.get('users')
if (user['id'] not in phaseAdminUserIds and
user['id'] != folder['creatorId'])]
for user in usersToRemove:
Folder().setUserAccess(folder, user, None)
# Add access to phase admins; ignore folder owner
usersToAdd = [user for user in phaseAdminUsers
if user['_id'] != folder['creatorId']]
for user in usersToAdd:
Folder().setUserAccess(folder, user, AccessType.READ)
# Save folder if access changed
if usersToRemove or usersToAdd:
Folder().save(folder, validate=False)
except TypeError:
raise ValidationException('A list of submissions is required.')
def create(
self,
tale: dict,
name: Optional[str],
versionsDir: Path,
versionsRoot: dict,
user=None,
force=False,
) -> dict:
last = self.getLastVersion(versionsRoot)
last_restore = Folder().load(tale.get("restoredFrom", ObjectId()),
force=True)
workspace = Folder().load(tale["workspaceId"], force=True)
crtWorkspace = Path(workspace["fsPath"])
# NOTE: order is important, we want oldWorkspace -> last.workspace
for version in (last_restore, last):
oldWorkspace = (None if version is None else
Path(version["fsPath"]) / "workspace")
if (not force and self.is_same(tale, version, user)
and self.sameTree(oldWorkspace, crtWorkspace)):
assert version is not None
raise RestException("Not modified",
code=303,
extra=str(version["_id"]))
new_version = self.createSubdir(versionsDir,
versionsRoot,
name,
user=user)
try:
self.snapshot(last, tale, new_version, user=user, force=force)
return new_version
except Exception: # NOQA
try:
shutil.rmtree(new_version["fsPath"])
Folder().remove(new_version)
except Exception as ex: # NOQA
logger.warning(
"Exception caught while rolling back version ckeckpoint.",
ex)
raise
def tearDown(self):
for folder in (self.public_folder, self.private_folder,
self.regular_folder):
Folder().remove(folder)
Collection().remove(self.base_collection)
for user in self.users.values():
User().remove(user)
for root in (self.public_root, self.private_root):
shutil.rmtree(root)
super(ItemOperationsTestCase, self).tearDown()
def getImageList(self, folderId, limit, offset):
printOk('getImageList() was called!')
limit, offset = int(limit), int(offset)
self.user = self.getCurrentUser()
folder = Folder().load(folderId,
level=AccessType.READ,
user=self.getCurrentUser())
items = Folder().childItems(folder, limit=limit, offset=offset)
items = self.__filter(items,
exts=[".jpg", ".jpeg", ".png", ".PNG", ".svs"])
ret_files = []
for item in items:
# TODO: remove this function
filename = os.path.splitext(item['name'])[0]
printOk("filename: " + filename)
ret_files.append(item)
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(ret_files)
def testTaleCopy(self):
from girder.plugins.wholetale.models.tale import Tale
from girder.plugins.wholetale.constants import TaleStatus
from girder.plugins.jobs.models.job import Job
from girder.plugins.jobs.constants import JobStatus
tale = Tale().createTale(self.image, [],
creator=self.admin,
public=True)
workspace = Tale().createWorkspace(tale)
# Below workarounds a bug, it will be addressed elsewhere.
workspace = Folder().setPublic(workspace, True, save=True)
adapter = assetstore_utilities.getAssetstoreAdapter(self.ws_assetstore)
size = 101
data = BytesIO(b' ' * size)
files = []
files.append(Upload().uploadFromFile(data,
size,
'file01.txt',
parentType='folder',
parent=workspace,
assetstore=self.ws_assetstore))
fullPath = adapter.fullPath(files[0])
# Create a copy
resp = self.request(path='/tale/{_id}/copy'.format(**tale),
method='POST',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertFalse(new_tale['public'])
self.assertEqual(new_tale['dataSet'], tale['dataSet'])
self.assertEqual(new_tale['copyOfTale'], str(tale['_id']))
self.assertEqual(new_tale['imageId'], str(tale['imageId']))
self.assertEqual(new_tale['creatorId'], str(self.user['_id']))
self.assertEqual(new_tale['status'], TaleStatus.PREPARING)
copied_file_path = re.sub(workspace['name'], new_tale['_id'], fullPath)
job = Job().findOne({'type': 'wholetale.copy_workspace'})
for i in range(10):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.SUCCESS:
break
time.sleep(0.1)
self.assertTrue(os.path.isfile(copied_file_path))
resp = self.request(path='/tale/{_id}'.format(**new_tale),
method='GET',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertEqual(new_tale['status'], TaleStatus.READY)
Tale().remove(new_tale)
Tale().remove(tale)
def filesResponse(self, path):
user = self.getCurrentUser()
path = '/'.join(path)
# Handle /collection and /user specially
if path in ('user', 'collection'):
if not cherrypy.request.path_info.endswith('/'):
return self._forward()
model = User() if path == 'user' else Collection()
return self._listDirectory(path, model.list(
user=user, sort=[('user' if path == 'user' else 'name', SortDir.ASCENDING)]))
resource = path_util.lookUpPath(path, user)['document']
singleFile = None
if resource['_modelType'] == 'file':
singleFile = resource
elif resource['_modelType'] == 'item':
singleFile = None
for _path, file in Item().fileList(doc=resource, user=user, subpath=True, data=False):
if singleFile is None:
singleFile = file
else:
singleFile = False
break
if singleFile is not False and singleFile is not None:
offset, endByte = 0, None
rangeHeader = cherrypy.lib.httputil.get_ranges(
cherrypy.request.headers.get('Range'), singleFile.get('size', 0))
if rangeHeader and len(rangeHeader):
offset, endByte = rangeHeader[0]
singleFile = File().load(singleFile['_id'], user=user, level=AccessType.READ)
return File().download(singleFile, offset, endByte=endByte)
if not cherrypy.request.path_info.endswith('/'):
return self._forward()
children = []
if resource['_modelType'] != 'item':
children.extend(Folder().childFolders(
parentType=resource['_modelType'], parent=resource, user=user,
sort=[('name', SortDir.ASCENDING)]))
if resource['_modelType'] == 'folder':
children.extend(Folder().childItems(resource, sort=[('name', SortDir.ASCENDING)]))
if resource['_modelType'] == 'item':
children.extend(Item().childFiles(resource, sort=[('name', SortDir.ASCENDING)]))
return self._listDirectory(path, children)
def testVirtualFolderValidation(self):
# Can't make folder virtual if it has children
subfolder = Folder().createFolder(self.f1, 'sub', creator=self.admin)
self.f1['isVirtual'] = True
with six.assertRaisesRegex(
self, ValidationException,
'Virtual folders may not contain child folders.'):
Folder().save(self.f1)
Folder().remove(subfolder)
item = Item().createItem('i', creator=self.admin, folder=self.f1)
with six.assertRaisesRegex(
self, ValidationException,
'Virtual folders may not contain child items.'):
Folder().save(self.f1)
Item().remove(item)
Folder().save(self.f1)
# Can't make subfolders or items under a virtual folder
with six.assertRaisesRegex(
self, ValidationException,
'You may not place items under a virtual folder.'):
Item().createItem('i', creator=self.admin, folder=self.f1)
with six.assertRaisesRegex(
self, ValidationException,
'You may not place folders under a virtual folder.'):
Folder().createFolder(self.f1, 'f', creator=self.admin)
# Can't move an item under a virtual folder
item = Item().createItem('i', creator=self.admin, folder=self.f2)
with six.assertRaisesRegex(
self, ValidationException,
'You may not place items under a virtual folder.'):
Item().move(item, self.f1)
# Ensure JSON for query
self.f1['virtualItemsQuery'] = 'not JSON'
with six.assertRaisesRegex(
self, ValidationException,
'The virtual items query must be valid JSON.'):
Folder().save(self.f1)
del self.f1['virtualItemsQuery']
self.f1['virtualItemsSort'] = 'not JSON'
with six.assertRaisesRegex(
self, ValidationException,
'The virtual items sort must be valid JSON.'):
Folder().save(self.f1)
def remove(self, calc, user=None, force=False):
super(Calculation, self).remove(calc)
# remove ingested file
file_id = calc.get('fileId')
if file_id is not None:
file = File().load(file_id, user=user, level=AccessType.WRITE)
if file:
item = Item().load(file['itemId'],
user=user,
level=AccessType.WRITE)
if item:
Item().remove(item)
# remove scratch folder with calculation output
scratch_folder_id = calc.get('scratchFolderId')
if scratch_folder_id is not None:
folder = Folder().load(scratch_folder_id,
user=user,
level=AccessType.WRITE)
if folder:
Folder().remove(folder)
def childItems(self, folder, limit=0, offset=0, sort=None, filters=None,
includeVirtual=False, **kwargs):
if not includeVirtual or not folder.get('isVirtual') or 'virtualItemsQuery' not in folder:
return Folder._childItemsBeforeHUI(
self, folder, limit=limit, offset=offset, sort=sort,
filters=filters, **kwargs)
q = json_util.loads(folder['virtualItemsQuery'])
if 'virtualItemsSort' in folder and sort is None:
sort = json.loads(folder['virtualItemsSort'])
q.update(filters or {})
return Item().find(q, limit=limit, offset=offset, sort=sort, **kwargs)
def saveImportAttributes(folder, attributes, user):
attributes_dict = fromMeta(folder, 'attributes', {})
# we don't overwrite any existing meta attributes
for attribute in attributes.values():
validated: models.Attribute = models.Attribute(**attribute)
if attribute['key'] not in attributes_dict:
attributes_dict[str(
validated.key)] = validated.dict(exclude_none=True)
folder['meta']['attributes'] = attributes_dict
Folder().save(folder)
def findSharedDatasetFolders(currentUser):
folderModel = Folder()
groupModel = ModelImporter.model('group')
datasetSharingGroup = groupModel.findOne(
query={'name': PluginSettings.DATASET_SHARING_GROUP_NAME})
if not datasetSharingGroup:
raise AccessException('user group "{0}" doesn\'t exist'.format(
PluginSettings.DATASET_SHARING_GROUP_NAME))
if datasetSharingGroup['_id'] not in currentUser['groups']:
raise AccessException(
'user doesn\'t belong to user group "{0}"'.format(
PluginSettings.DATASET_SHARING_GROUP_NAME))
folders = folderModel.find({
'baseParentType': 'user',
'parentCollection': 'user',
'access.groups.id': datasetSharingGroup['_id'],
'name': PluginSettings.MINERVA_SHARED_DATASET
})
return folders
def create_folder(self, event, path, root, user=None):
params = event.info["params"]
new_path = path / params["name"]
try:
new_path.mkdir()
except FileExistsError:
raise ValidationException(
"A folder with that name already exists here.", "name")
event.preventDefault().addResponse(Folder().filter(self.vFolder(
new_path, root),
user=user))
def training_output_folder(user: types.GirderUserModel) -> types.GirderModel:
"""Ensure that the user has a training results folder."""
viameFolder = Folder().createFolder(
user,
constants.ViameDataFolderName,
description="VIAME data storage.",
parentType="user",
public=False,
creator=user,
reuseExisting=True,
)
return Folder().createFolder(
viameFolder,
constants.TrainingOutputFolderName,
description="Results from VIAME model training are placed here.",
public=False,
creator=user,
reuseExisting=True,
)
def testPrivateUser(self):
"""
Make sure private users behave correctly.
"""
# Create an admin user
User().createUser(firstName='Admin',
lastName='Admin',
login='******',
email='*****@*****.**',
password='******')
# Register a private user (non-admin)
pvt = User().createUser(firstName='Guy',
lastName='Noir',
login='******',
email='*****@*****.**',
password='******',
public=False)
self.assertEqual(pvt['public'], False)
folder = six.next(Folder().childFolders(parentType='user', parent=pvt))
# Private users should be able to upload files
resp = self.request(path='/item',
method='POST',
user=pvt,
params={
'name': 'foo.txt',
'folderId': folder['_id']
})
self.assertStatusOk(resp)
itemId = resp.json['_id']
resp = self.request(path='/file',
method='POST',
user=pvt,
params={
'parentType': 'item',
'parentId': itemId,
'name': 'foo.txt',
'size': 5,
'mimeType': 'text/plain'
})
self.assertStatusOk(resp)
fields = [('offset', 0), ('uploadId', resp.json['_id'])]
files = [('chunk', 'foo.txt', 'hello')]
resp = self.multipartRequest(path='/file/chunk',
user=pvt,
fields=fields,
files=files)
self.assertStatusOk(resp)
self.assertEqual(resp.json['itemId'], itemId)
def _folderUpdate(self, event):
params = event.info['params']
if {'isVirtual', 'virtualItemsQuery', 'virtualItemsSort'} & set(params):
folder = Folder().load(event.info['returnVal']['_id'], force=True)
update = False
if params.get('isVirtual') is not None:
update = True
folder['isVirtual'] = params['isVirtual']
if params.get('virtualItemsQuery') is not None:
update = True
folder['virtualItemsQuery'] = params['virtualItemsQuery']
if params.get('virtualItemsSort') is not None:
update = True
folder['virtualItemsSort'] = params['virtualItemsSort']
if update:
self.requireAdmin(self.getCurrentUser(), 'Must be admin to setup virtual folders.')
folder = Folder().filter(Folder().save(folder), rest.getCurrentUser())
event.preventDefault().addResponse(folder)
def setStatus(self, rfolder: dict, status: Union[int, RunState]) -> None:
# TODO: add heartbeats (runs must regularly update status, otherwise they are considered
# failed)
if isinstance(status, int):
_status = RunState.ALL[status]
else:
_status = status
rfolder[FIELD_STATUS_CODE] = _status.code
Folder().save(rfolder)
runDir = Path(rfolder["fsPath"])
self.write_status(runDir, _status)
def drafts_collection(db):
red_herring_collection = Collection().createCollection(
"red_herring_collection", reuseExisting=True)
# A folder that matches dandiset metadata, but not in drafts collection.
red_herring_dandiset_000001_folder = Folder().createFolder(
parent=red_herring_collection,
parentType="collection",
name="000001",
public=True,
)
meta = {
"dandiset": {
"name": "red",
"description": "herring",
"identifier": "000001"
}
}
Folder().setMetadata(red_herring_dandiset_000001_folder, meta)
return get_or_create_drafts_collection()
def check_existing_annotations(event):
info = event.info
if "annotations.csv" in info["importPath"]:
item = Item().findOne({"_id": info["id"]})
item["meta"].update({
"folderId": str(item["folderId"]),
"pipeline": None
})
Item().save(item)
folder = Folder().findOne({"_id": item["folderId"]})
# FPS is hardcoded for now
folder["meta"].update({
"type": "image-sequence",
"viame": True,
"fps": 30
})
Folder().save(folder)
def parentChain(admin):
# Create the parent chain
F1 = Folder().createFolder(
parent=admin, parentType='user', creator=admin,
name='F1', public=True)
F2 = Folder().createFolder(
parent=F1, parentType='folder', creator=admin,
name='F2', public=True)
privateFolder = Folder().createFolder(
parent=F2, parentType='folder', creator=admin,
name='F3', public=False)
F4 = Folder().createFolder(
parent=privateFolder, parentType='folder', creator=admin,
name='F4', public=True)
yield {
'folder1': F1,
'folder2': F2,
'privateFolder': privateFolder,
'folder4': F4
}
def getOrCreateRootFolder(name):
collection = Collection().createCollection(
name, public=True, reuseExisting=True)
# For backward compat
if not collection['public']:
collection = Collection().save(
Collection().setPublic(collection, True)
)
folder = Folder().createFolder(
collection, name, parentType='collection', public=True, reuseExisting=True)
return folder
def _testUploadWithPath(self):
testUser = User().createUser(firstName='Jeffrey',
lastName='Abrams',
login='******',
password='******',
email='*****@*****.**')
publicFolder = six.next(Folder().childFolders(parentType='user',
parent=testUser,
user=None,
limit=1))
self.client.upload(self.libTestDir, '/user/jjabrams/Public')
parent = six.next(Folder().childFolders(parentType='folder',
parent=publicFolder,
user=testUser,
limit=0))
self.assertEqual([
f['name'] for f in Folder().childFolders(
parentType='folder', parent=parent, user=testUser, limit=0)
], ['sub0', 'sub1', 'sub2'])
def import_recursive(job):
try:
root = job['kwargs']['root']
token = job['kwargs']['token']
user = User().load(job['userId'], force=True)
children = list(Folder().childFolders(root, 'collection', user=user))
count = len(children)
progress = 0
job = Job().updateJob(job,
log='Started TCGA import\n',
status=JobStatus.RUNNING,
progressCurrent=progress,
progressTotal=count)
logger.info('Starting recursive TCGA import')
for child in children:
progress += 1
try:
msg = 'Importing "%s"' % child.get('name', '')
job = Job().updateJob(job,
log=msg,
progressMessage=msg + '\n',
progressCurrent=progress)
logger.debug(msg)
Cohort().importDocument(child,
recurse=True,
user=user,
token=token,
job=job)
job = Job().load(id=job['_id'], force=True)
# handle any request to stop execution
if (not job or job['status']
in (JobStatus.CANCELED, JobStatus.ERROR)):
logger.info('TCGA import job halted with')
return
except ValidationException:
logger.warning('Failed to import %s' % child.get('name', ''))
logger.info('Starting recursive TCGA import')
job = Job().updateJob(job,
log='Finished TCGA import\n',
status=JobStatus.SUCCESS,
progressCurrent=count,
progressMessage='Finished TCGA import')
except Exception as e:
logger.exception('Importing TCGA failed with %s' % str(e))
job = Job().updateJob(job,
log='Import failed with %s\n' % str(e),
status=JobStatus.ERROR)
def _pruneOrphans(self, progress):
count = 0
models = [File(), Folder(), Item()]
steps = sum(model.find().count() for model in models)
progress.update(total=steps, current=0)
for model in models:
for doc in model.find():
progress.update(increment=1)
if model.isOrphan(doc):
model.remove(doc)
count += 1
return count
def processNotification(store: AssetstoreModel, rootFolder: GirderModel,
importPath: str):
"""
Import at proper location
"""
if rootFolder is None:
raise RestException('Root folder missing')
# Find the correct import location given rootFolder and importPath
owner = User().findOne({
'_id':
ObjectId(rootFolder['creatorId'] or rootFolder['baseParentId'])
})
if owner is None:
raise RestException('Root has no owner')
target = rootFolder
importPath = importPath.lstrip('/')
realImportPathDir, _ = os.path.split(importPath)
storePrefix = str(store['prefix']).lstrip('/')
common_path = os.path.commonpath([storePrefix, importPath])
realImportPath = common_path
path_from_root = importPath[len(common_path):].lstrip('/')
path_from_root_list, _ = os.path.split(path_from_root)
for folder_name in path_from_root_list.split('/'):
new_target = Folder().findOne({
'parentId': ObjectId(target['_id']),
'name': folder_name,
})
new_import_path = f'{realImportPath}/{folder_name}'
if new_target is not None:
target = new_target
realImportPath = new_import_path
else:
break
realImportPath = realImportPath.lstrip('/')
if realImportPath == realImportPathDir:
# All the chain of parent directories exist
realImportPath = importPath
Assetstore().importData(
store,
target,
'folder',
{'importPath': realImportPath},
None,
owner,
force_recursive=False,
)
def __init__(self):
super().__init__()
self.coll_m = Collection()
self.file_m = File()
self.folder_m = Folder()
self.item_m = Item()
self.upload_m = Upload()
self.asset_m = Assetstore()
self.user_m = User()
self.setupRoutes()
def importSlide(self, params):
user = self.getCurrentUser()
token = self.getCurrentToken()
self.requireParams('folderId', params)
folder = Folder().load(id=params['folderId'],
user=user,
level=AccessType.WRITE,
exc=True)
slide = Slide().importDocument(folder, user=user, token=token)
return slide
def _virtualChildItems(self, event):
params = event.info['params']
if 'folderId' not in params:
return # This is not a child listing request
user = self.getCurrentUser()
folder = Folder().load(params['folderId'], user=user, level=AccessType.READ)
if not folder.get('isVirtual') or 'virtualItemsQuery' not in folder:
return # Parent is not a virtual folder, proceed as normal
limit, offset, sort = self.getPagingParameters(params, defaultSortField='name')
q = json_util.loads(folder['virtualItemsQuery'])
if 'virtualItemsSort' in folder:
sort = json.loads(folder['virtualItemsSort'])
item = Item()
# These items may reside in folders that the user cannot read, so we must filter
items = item.filterResultsByPermission(
item.find(q, sort=sort), user, level=AccessType.READ, limit=limit, offset=offset)
event.preventDefault().addResponse([item.filter(i, user) for i in items])
def _validateFolder(event):
doc = event.info
if 'isVirtual' in doc and not isinstance(doc['isVirtual'], bool):
raise ValidationException('The isVirtual field must be boolean.', field='isVirtual')
if doc.get('isVirtual'):
# Make sure it doesn't have children
if list(Folder().childItems(doc, limit=1)):
raise ValidationException(
'Virtual folders may not contain child items.', field='isVirtual')
if list(Folder().find({
'parentId': doc['_id'],
'parentCollection': 'folder'
}, limit=1)):
raise ValidationException(
'Virtual folders may not contain child folders.', field='isVirtual')
if doc['parentCollection'] == 'folder':
parent = Folder().load(event.info['parentId'], force=True, exc=True)
if parent.get('isVirtual'):
raise ValidationException(
'You may not place folders under a virtual folder.', field='folderId')
if 'virtualItemsQuery' in doc:
try:
json.loads(doc['virtualItemsQuery'])
except (TypeError, ValueError):
raise ValidationException(
'The virtual items query must be valid JSON.', field='virtualItemsQuery')
if 'virtualItemsSort' in doc:
try:
json.loads(doc['virtualItemsSort'])
except (TypeError, ValueError):
raise ValidationException(
'The virtual items sort must be valid JSON.', field='virtualItemsSort')
def __init__(self):
super(Folder, self).__init__()
self.resourceName = 'folder'
self._model = FolderModel()
self.route('DELETE', (':id',), self.deleteFolder)
self.route('DELETE', (':id', 'contents'), self.deleteContents)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getFolder)
self.route('GET', (':id', 'details'), self.getFolderDetails)
self.route('GET', (':id', 'access'), self.getFolderAccess)
self.route('GET', (':id', 'download'), self.downloadFolder)
self.route('GET', (':id', 'rootpath'), self.rootpath)
self.route('POST', (), self.createFolder)
self.route('PUT', (':id',), self.updateFolder)
self.route('PUT', (':id', 'access'), self.updateFolderAccess)
self.route('POST', (':id', 'copy'), self.copyFolder)
self.route('PUT', (':id', 'metadata'), self.setMetadata)
self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)
class AssignmentResource(Resource):
def __init__(self):
super().__init__()
self.coll_m = Collection()
self.file_m = File()
self.folder_m = Folder()
self.item_m = Item()
self.upload_m = Upload()
self.asset_m = Assetstore()
self.user_m = User()
self.setupRoutes()
def setupRoutes(self):
self.route('GET', (), handler=self.list)
self.route('GET', (':a_id',), handler=self.getAssignment)
self.route('GET', ('admin_data',), handler=self.getAdminData)
def __findName(self, folder):
imageFolder = self.__findImageFolder(folder)
if isinstance(imageFolder, dict):
return imageFolder['name']
return ""
def __findImageFolder(self, folder):
owner = self.__findOwner(folder)
ret = ""
if self.folder_m.getAccessLevel(folder, self.getCurrentUser()) == AccessType.ADMIN:
# this folder was created by this user, and so it will contain the images
# printFail(folder)
ret = folder
else:
meta = folder['meta']
# this is the label file, and so should only have one entry in the metadata
assert len(meta) == 1
# that one entry contains link to the image folder, key must be the creator of this folder
assert str(owner['_id']) in meta, (str(owner['_id']), meta)
ret = self.folder_m.load(meta[str(owner['_id'])],
level=AccessType.READ,
user=self.getCurrentUser())
return ret
def __findLabelFolder(self, folder):
owner = self.__findOwner(folder)
ret = []
if self.folder_m.getAccessLevel(folder, self.getCurrentUser()) == AccessType.ADMIN:
# this folder was created by this user, so it will contain images
if 'meta' not in folder:
return []
meta = folder['meta']
for k, v in meta.items():
ret.append(self.folder_m.load(v, level=AccessType.READ, user=self.getCurrentUser()))
else:
# this folder was not created by this user, so it will contain labels
ret = [folder]
return ret
def __findOwner(self, folder):
aclList = Folder().getFullAccessList(folder)
for acl in aclList['users']:
if acl['level'] == AccessType.ADMIN:
return self.user_m.load(str(acl['id']), level=AccessType.READ, user=self.getCurrentUser())
return None
@access.public
@autoDescribeRoute(
Description('Get list of all assignments that it owns or is a part of')
.param('limit', 'Number of assignments to return')
.param('offset', 'offset from 0th assignment to start looking for assignments'))
@rest.rawResponse
def list(self, limit, offset):
try:
printOk((limit, offset))
ret = self.__list(int(limit), int(offset))
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(ret)
except:
printFail(traceback.print_exc)
def __list(self, limit=0, offset=0):
user = self.getCurrentUser()
folders = self.folder_m.find({'parentId': user['_id'],
'parentCollection': 'user'}, limit=limit, offset=offset)
ret = []
for folder in folders:
if self.__findName(folder):
ret.append({'name': self.__findName(folder),
'image_folder': self.__findImageFolder(folder),
'label_folders': self.__findLabelFolder(folder),
'owner': self.__findOwner(folder)})
return ret
def __getAssignment(self, _id):
assignments = self.__list()
for assignment in assignments:
# printOk2(assignment)
if str(assignment['image_folder']['_id']) == _id:
return assignment
return None
def __getAnnotators(self, assignment):
ret = []
for label_folder in assignment['label_folders']:
printOk(label_folder)
ret.append({
'user': self.user_m.load(label_folder['parentId'], user=self.getCurrentUser(), level=AccessType.READ),
'expanded': False,
})
return ret
@access.public
@autoDescribeRoute(
Description('Get assignment by id').param('a_id', 'folder id that controls the assignment'))
@rest.rawResponse
def getAssignment(self, a_id):
try:
assignment = self.__getAssignment(a_id)
return dumps(assignment)
except:
printFail(traceback.print_exc)
@access.public
@autoDescribeRoute(
Description('Get admin data associated with assignment id').param('a_id',
'folder id that controls the assignment'))
@rest.rawResponse
def getAdminData(self, a_id):
try:
assignment = self.__getAssignment(a_id)
if assignment['owner']['_id'] == self.getCurrentUser()['_id']:
# then current user is this assignment's admin
return dumps({'annotators': self.__getAnnotators(assignment)})
except:
printFail(traceback.print_exc)
class LabelResource(Resource):
def __init__(self):
super().__init__()
self.resourceName = 'label'
self.coll_m = Collection()
self.file_m = File()
self.folder_m = Folder()
self.item_m = Item()
self.upload_m = Upload()
self.asset_m = Assetstore()
self.setupRoutes()
def setupRoutes(self):
self.route('GET', (), handler=self.getLabelList)
self.route('GET', (':label_id',), self.getLabel)
self.route('GET', ('meta',), self.getLabelMeta)
self.route('GET', ('create',), self.createLabelFile)
self.route('GET', ('by_name',), self.getLabelByName)
self.route('POST', (), self.postLabel)
def createNewFile(self, folder, file_name):
item = self.item_m.createItem(file_name,
creator=self.getCurrentUser(),
folder=folder,
description='label file',
reuseExisting=False)
file = self.file_m.createFile(size=0,
item=item,
name=file_name,
creator=self.getCurrentUser(),
assetstore=self.asset_m.getCurrent(),
mimeType="application/json")
return file
def copy(self, srcFile, destFile):
upload = self.upload_m.createUploadToFile(destFile, self.getCurrentUser(), srcFile['size'])
self.upload_m.handleChunk(upload=upload,
chunk=RequestBodyStream(self.file_m.open(srcFile), size=destFile['size']),
user=self.getCurrentUser())
return upload
@access.public
@autoDescribeRoute(
Description('Get label list'))
@rest.rawResponse
def getLabelList(self):
printOk('getLabelsList() was called!')
try:
collection = list(self.coll_m.list(user=self.getCurrentUser(), offset=0, limit=1))[0]
files = self.coll_m.fileList(collection, user=self.getCurrentUser(), data=False,
includeMetadata=True, mimeFilter=['application/json'])
files = list(files)
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(files)
except:
printFail(traceback.print_exc)
@staticmethod
def getOwnerId(folder):
aclList = Folder().getFullAccessList(folder)
for acl in aclList['users']:
if acl['level'] == AccessType.ADMIN:
return str(acl['id'])
return None
def getConfigFolder(self, label_folder_id):
label_folder = Folder().load(label_folder_id,
user=self.getCurrentUser(),
level=AccessType.READ)
ownerId = self.getOwnerId(label_folder)
config_folder = self.folder_m.load(label_folder['meta'][ownerId], level=AccessType.READ,
user=self.getCurrentUser())
return config_folder
def findConfig(self, folder_id):
folder = self.getConfigFolder(folder_id)
printOk2("Config folder {}".format(folder))
files = self.folder_m.fileList(folder, self.getCurrentUser(), data=False)
for file_path, file in files:
printOk(file)
if file['name'] == "config.json":
return file
def __findFile(self, folder, file_name):
item = list(self.item_m.find({'folderId': folder['_id'], 'name': file_name}).limit(1))
if not item:
return None
item = item[0]
file = list(self.file_m.find({'itemId': item['_id']}).limit(1))
if not file:
return None
return file[0]
@access.public
@autoDescribeRoute(
Description('Create a new label file if it doesnt exist')
.param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
@rest.rawResponse
def createLabelFile(self, file_name, folder_id):
try:
folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.WRITE)
file = self.__findFile(folder, file_name)
if not file:
file = self.createNewFile(folder, file_name)
config_file = self.findConfig(folder_id)
if not config_file:
printFail("No config file found")
return errorMessage("No config file found")
else:
res = self.copy(config_file, file)
return dumps({
"label_id": res['fileId']
})
return dumps({
"label_id": file['_id']
})
except:
printFail(traceback.print_exc)
cherrypy.response.status = 500
@access.public
@autoDescribeRoute(
Description('Get labels by file_name')
.param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
@rest.rawResponse
def getLabelByName(self, file_name, folder_id):
try:
folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.READ)
file = self.__findFile(folder, file_name)
cherrypy.response.headers["Content-Type"] = "application/json"
if file:
return self.file_m.download(file)
else:
return dumps({})
except:
printFail(traceback.print_exc)
cherrypy.response.status = 500
@access.public
@autoDescribeRoute(
Description('Get label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def getLabel(self, label_id):
try:
file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
printOk2(file)
cherrypy.response.headers["Content-Type"] = "application/json"
return self.file_m.download(file)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Get label by id')
.param('label_id', 'label file id'))
def getLabelMeta(self, label_id):
try:
file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(file)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Post label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def postLabel(self, label_id, params):
try:
file = self.file_m.load(label_id, level=AccessType.WRITE, user=self.getCurrentUser())
cherrypy.response.headers["Content-Type"] = "application/json"
params['labels'] = json.loads(params['labels'])
data = json.dumps(params, indent=2, sort_keys=True)
upload = writeData(self.getCurrentUser(), file, data)
printOk2(file)
printOk(upload)
return dumps(upload)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Post label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def strokeToOutline(self, strokes):
pass
def setCuration(self, folder, params):
user = self.getCurrentUser()
if CURATION not in folder:
folder[CURATION] = dict(DEFAULTS)
curation = folder[CURATION]
oldCuration = dict(curation)
# update enabled
oldEnabled = curation.get(ENABLED, False)
if ENABLED in params:
self.requireAdmin(user)
curation[ENABLED] = self.boolParam(ENABLED, params)
enabled = curation.get(ENABLED, False)
# update status
oldStatus = curation.get(STATUS)
if STATUS in params:
# verify valid status
if params[STATUS] not in [CONSTRUCTION, REQUESTED, APPROVED]:
raise RestException('Invalid status parameter')
# regular users can only do construction -> requested transition
if curation.get(STATUS) != CONSTRUCTION or \
params[STATUS] != REQUESTED:
self.requireAdmin(user)
curation[STATUS] = params[STATUS]
status = curation.get(STATUS)
# admin enabling curation
if enabled and not oldEnabled:
# TODO: check if writers exist?
# TODO: email writers?
with self._progressContext(folder, TITLE_PRIVATE) as pc:
self._setPublic(folder, False, pc)
curation[ENABLE_USER_ID] = user.get('_id')
self._addTimeline(oldCuration, curation, 'enabled curation')
# admin reopening folder
if enabled and oldStatus == APPROVED and status == CONSTRUCTION:
with self._progressContext(folder, TITLE_PRIVATE) as pc:
self._setPublic(folder, False, pc)
curation[ENABLE_USER_ID] = user.get('_id')
with self._progressContext(folder, TITLE_WRITEABLE) as pc:
self._makeWriteable(folder, pc)
self._addTimeline(oldCuration, curation, 'reopened folder')
# admin disabling curation
if not enabled and oldEnabled:
self._addTimeline(oldCuration, curation, 'disabled curation')
# user requesting approval
if enabled and oldStatus == CONSTRUCTION and status == REQUESTED:
curation[REQUEST_USER_ID] = user.get('_id')
with self._progressContext(folder, TITLE_READ_ONLY) as pc:
self._makeReadOnly(folder, pc)
self._addTimeline(oldCuration, curation, 'requested approval')
# send email to admin requesting approval
self._sendMail(
folder, curation.get(ENABLE_USER_ID),
'REQUEST FOR APPROVAL: ' + folder['name'],
'curation.requested.mako')
# admin approving request
if enabled and oldStatus == REQUESTED and status == APPROVED:
with self._progressContext(folder, TITLE_PUBLIC) as pc:
self._setPublic(folder, True, pc)
curation[REVIEW_USER_ID] = user.get('_id')
self._addTimeline(oldCuration, curation, 'approved request')
# send approval notification to requestor
self._sendMail(
folder, curation.get(REQUEST_USER_ID),
'APPROVED: ' + folder['name'],
'curation.approved.mako')
# admin rejecting request
if enabled and oldStatus == REQUESTED and status == CONSTRUCTION:
curation[REVIEW_USER_ID] = user.get('_id')
with self._progressContext(folder, TITLE_WRITEABLE) as pc:
self._makeWriteable(folder, pc)
self._addTimeline(oldCuration, curation, 'rejected request')
# send rejection notification to requestor
self._sendMail(
folder, curation.get(REQUEST_USER_ID),
'REJECTED: ' + folder['name'],
'curation.rejected.mako')
folder = Folder().save(folder)
curation['public'] = folder.get('public')
return curation
def validate(self, doc, **kwargs):
# TODO: implement
# raise ValidationException
return Folder.validate(self, doc, **kwargs)
def findOne(self, query=None, annotator_user=None, state=None, **kwargs):
study_query = self._find_query_filter(query, annotator_user, state)
return Folder.findOne(self, study_query, **kwargs)
class Folder(Resource):
"""API Endpoint for folders."""
def __init__(self):
super(Folder, self).__init__()
self.resourceName = 'folder'
self._model = FolderModel()
self.route('DELETE', (':id',), self.deleteFolder)
self.route('DELETE', (':id', 'contents'), self.deleteContents)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getFolder)
self.route('GET', (':id', 'details'), self.getFolderDetails)
self.route('GET', (':id', 'access'), self.getFolderAccess)
self.route('GET', (':id', 'download'), self.downloadFolder)
self.route('GET', (':id', 'rootpath'), self.rootpath)
self.route('POST', (), self.createFolder)
self.route('PUT', (':id',), self.updateFolder)
self.route('PUT', (':id', 'access'), self.updateFolderAccess)
self.route('POST', (':id', 'copy'), self.copyFolder)
self.route('PUT', (':id', 'metadata'), self.setMetadata)
self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Search for folders by certain properties.')
.notes('You must pass either a "folderId" or "text" field '
'to specify how you are searching for folders. '
'If you omit one of these parameters the request will fail and respond : '
'"Invalid search mode."')
.responseClass('Folder', array=True)
.param('parentType', "Type of the folder's parent", required=False,
enum=['folder', 'user', 'collection'])
.param('parentId', "The ID of the folder's parent.", required=False)
.param('text', 'Pass to perform a text search.', required=False)
.param('name', 'Pass to lookup a folder by exact name match. Must '
'pass parentType and parentId as well when using this.', required=False)
.pagingParams(defaultSort='lowerName')
.errorResponse()
.errorResponse('Read access was denied on the parent resource.', 403)
)
def find(self, parentType, parentId, text, name, limit, offset, sort):
"""
Get a list of folders with given search parameters. Currently accepted
search modes are:
1. Searching by parentId and parentType, with optional additional
filtering by the name field (exact match) or using full text search
within a single parent folder. Pass a "name" parameter or "text"
parameter to invoke these additional filters.
2. Searching with full text search across all folders in the system.
Simply pass a "text" parameter for this mode.
"""
user = self.getCurrentUser()
if parentType and parentId:
parent = ModelImporter.model(parentType).load(
parentId, user=user, level=AccessType.READ, exc=True)
filters = {}
if text:
filters['$text'] = {
'$search': text
}
if name:
filters['name'] = name
return self._model.childFolders(
parentType=parentType, parent=parent, user=user,
offset=offset, limit=limit, sort=sort, filters=filters)
elif text:
return self._model.textSearch(
text, user=user, limit=limit, offset=offset, sort=sort)
else:
raise RestException('Invalid search mode.')
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Get detailed information about a folder.')
.modelParam('id', model=FolderModel, level=AccessType.READ)
.errorResponse()
.errorResponse('Read access was denied on the folder.', 403)
)
def getFolderDetails(self, folder):
return {
'nItems': self._model.countItems(folder),
'nFolders': self._model.countFolders(
folder, user=self.getCurrentUser(), level=AccessType.READ)
}
@access.public(scope=TokenScope.DATA_READ, cookie=True)
@autoDescribeRoute(
Description('Download an entire folder as a zip archive.')
.modelParam('id', model=FolderModel, level=AccessType.READ)
.jsonParam('mimeFilter', 'JSON list of MIME types to include.', required=False,
requireArray=True)
.produces('application/zip')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the folder.', 403)
)
def downloadFolder(self, folder, mimeFilter):
"""
Returns a generator function that will be used to stream out a zip
file containing this folder's contents, filtered by permissions.
"""
setResponseHeader('Content-Type', 'application/zip')
setContentDisposition(folder['name'] + '.zip')
user = self.getCurrentUser()
def stream():
zip = ziputil.ZipGenerator(folder['name'])
for (path, file) in self._model.fileList(
folder, user=user, subpath=False, mimeFilter=mimeFilter):
for data in zip.addFile(file, path):
yield data
yield zip.footer()
return stream
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Update a folder or move it into a new parent.')
.responseClass('Folder')
.modelParam('id', model=FolderModel, level=AccessType.WRITE)
.param('name', 'Name of the folder.', required=False, strip=True)
.param('description', 'Description for the folder.', required=False, strip=True)
.param('parentType', "Type of the folder's parent", required=False,
enum=['folder', 'user', 'collection'], strip=True)
.param('parentId', 'Parent ID for the new parent of this folder.', required=False)
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='form', requireObject=True, required=False)
.errorResponse('ID was invalid.')
.errorResponse('Write access was denied for the folder or its new parent object.', 403)
)
def updateFolder(self, folder, name, description, parentType, parentId, metadata):
user = self.getCurrentUser()
if name is not None:
folder['name'] = name
if description is not None:
folder['description'] = description
folder = self._model.updateFolder(folder)
if metadata:
folder = self._model.setMetadata(folder, metadata)
if parentType and parentId:
parent = ModelImporter.model(parentType).load(
parentId, level=AccessType.WRITE, user=user, exc=True)
if (parentType, parent['_id']) != (folder['parentCollection'], folder['parentId']):
folder = self._model.move(folder, parent, parentType)
return folder
@access.user(scope=TokenScope.DATA_OWN)
@filtermodel(model=FolderModel, addFields={'access'})
@autoDescribeRoute(
Description('Update the access control list for a folder.')
.modelParam('id', model=FolderModel, level=AccessType.ADMIN)
.jsonParam('access', 'The JSON-encoded access control list.', requireObject=True)
.jsonParam('publicFlags', 'JSON list of public access flags.', requireArray=True,
required=False)
.param('public', 'Whether the folder should be publicly visible.',
dataType='boolean', required=False)
.param('recurse', 'Whether the policies should be applied to all '
'subfolders under this folder as well.', dataType='boolean',
default=False, required=False)
.param('progress', 'If recurse is set to True, this controls whether '
'progress notifications will be sent.', dataType='boolean',
default=False, required=False)
.errorResponse('ID was invalid.')
.errorResponse('Admin access was denied for the folder.', 403)
)
def updateFolderAccess(self, folder, access, publicFlags, public, recurse, progress):
user = self.getCurrentUser()
progress = progress and recurse # Only enable progress in recursive case
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self._model.subtreeCount(
folder, includeItems=False, user=user, level=AccessType.ADMIN))
return self._model.setAccessList(
folder, access, save=True, recurse=recurse, user=user,
progress=ctx, setPublic=public, publicFlags=publicFlags)
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Create a new folder.')
.responseClass('Folder')
.param('parentType', "Type of the folder's parent", required=False,
enum=['folder', 'user', 'collection'], default='folder')
.param('parentId', "The ID of the folder's parent.")
.param('name', 'Name of the folder.', strip=True)
.param('description', 'Description for the folder.', required=False,
default='', strip=True)
.param('reuseExisting', 'Return existing folder if it exists rather than '
'creating a new one.', required=False,
dataType='boolean', default=False)
.param('public', 'Whether the folder should be publicly visible. By '
'default, inherits the value from parent folder, or in the '
'case of user or collection parentType, defaults to False.',
required=False, dataType='boolean')
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='form', requireObject=True, required=False)
.errorResponse()
.errorResponse('Write access was denied on the parent', 403)
)
def createFolder(self, public, parentType, parentId, name, description,
reuseExisting, metadata):
user = self.getCurrentUser()
parent = ModelImporter.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
newFolder = self._model.createFolder(
parent=parent, name=name, parentType=parentType, creator=user,
description=description, public=public, reuseExisting=reuseExisting)
if metadata:
newFolder = self._model.setMetadata(newFolder, metadata)
return newFolder
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Get a folder by ID.')
.responseClass('Folder')
.modelParam('id', model=FolderModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the folder.', 403)
)
def getFolder(self, folder):
return folder
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Get the access control list for a folder.')
.responseClass('Folder')
.modelParam('id', model=FolderModel, level=AccessType.ADMIN)
.errorResponse('ID was invalid.')
.errorResponse('Admin access was denied for the folder.', 403)
)
def getFolderAccess(self, folder):
return self._model.getFullAccessList(folder)
@access.user(scope=TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Delete a folder by ID.')
.modelParam('id', model=FolderModel, level=AccessType.ADMIN)
.param('progress', 'Whether to record progress on this task.',
required=False, dataType='boolean', default=False)
.errorResponse('ID was invalid.')
.errorResponse('Admin access was denied for the folder.', 403)
)
def deleteFolder(self, folder, progress):
with ProgressContext(progress, user=self.getCurrentUser(),
title='Deleting folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder))
self._model.remove(folder, progress=ctx)
return {'message': 'Deleted folder %s.' % folder['name']}
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Set metadata fields on an folder.')
.responseClass('Folder')
.notes('Set metadata fields to null in order to delete them.')
.modelParam('id', model=FolderModel, level=AccessType.WRITE)
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='body', requireObject=True)
.param('allowNull', 'Whether "null" is allowed as a metadata value.', required=False,
dataType='boolean', default=False)
.errorResponse(('ID was invalid.',
'Invalid JSON passed in request body.',
'Metadata key name was invalid.'))
.errorResponse('Write access was denied for the folder.', 403)
)
def setMetadata(self, folder, metadata, allowNull):
return self._model.setMetadata(folder, metadata, allowNull=allowNull)
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=FolderModel)
@autoDescribeRoute(
Description('Copy a folder.')
.responseClass('Folder')
.modelParam('id', 'The ID of the original folder.', model=FolderModel,
level=AccessType.READ)
.param('parentType', "Type of the new folder's parent", required=False,
enum=['folder', 'user', 'collection'])
.param('parentId', 'The ID of the parent document.', required=False)
.param('name', 'Name for the new folder.', required=False)
.param('description', "Description for the new folder.", required=False)
.param('public', "Whether the folder should be publicly visible. By "
"default, inherits the value from parent folder, or in the case "
"of user or collection parentType, defaults to False. If "
"'original', use the value of the original folder.",
required=False, enum=['true', 'false', 'original'])
.param('progress', 'Whether to record progress on this task.',
required=False, dataType='boolean', default=False)
.errorResponse(('A parameter was invalid.',
'ID was invalid.'))
.errorResponse('Read access was denied on the original folder.\n\n'
'Write access was denied on the parent.', 403)
)
def copyFolder(self, folder, parentType, parentId, name, description, public, progress):
user = self.getCurrentUser()
parentType = parentType or folder['parentCollection']
if parentId:
parent = ModelImporter.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
else:
parent = None
with ProgressContext(progress, user=self.getCurrentUser(),
title='Copying folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder))
return self._model.copyFolder(
folder, creator=user, name=name, parentType=parentType,
parent=parent, description=description, public=public, progress=ctx)
@access.user(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Remove all contents from a folder.')
.notes('Cleans out all the items and subfolders from under a folder, '
'but does not remove the folder itself.')
.modelParam('id', 'The ID of the folder to clean.', model=FolderModel,
level=AccessType.WRITE)
.param('progress', 'Whether to record progress on this task.',
required=False, dataType='boolean', default=False)
.errorResponse('ID was invalid.')
.errorResponse('Write access was denied on the folder.', 403)
)
def deleteContents(self, folder, progress):
with ProgressContext(progress, user=self.getCurrentUser(),
title='Clearing folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder) - 1)
self._model.clean(folder, progress=ctx)
return {'message': 'Cleaned folder %s.' % folder['name']}
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(FolderModel)
@autoDescribeRoute(
Description('Delete metadata fields on a folder.')
.responseClass('Folder')
.modelParam('id', model=FolderModel, level=AccessType.WRITE)
.jsonParam(
'fields', 'A JSON list containing the metadata fields to delete',
paramType='body', schema={
'type': 'array',
'items': {
'type': 'string'
}
}
)
.errorResponse(('ID was invalid.',
'Invalid JSON passed in request body.',
'Metadata key name was invalid.'))
.errorResponse('Write access was denied for the folder.', 403)
)
def deleteMetadata(self, folder, fields):
return self._model.deleteMetadata(folder, fields)
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Get the path to the root of the folder\'s hierarchy.')
.modelParam('id', model=FolderModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the folder.', 403)
)
def rootpath(self, folder, params):
return self._model.parentsToRoot(folder, user=self.getCurrentUser())
def findOne(self, query=None, **kwargs):
dataset_query = self._find_query_filter(query)
return Folder.findOne(self, dataset_query, **kwargs)
def _validateItem(event):
parent = Folder().load(event.info['folderId'], force=True, exc=True)
if parent.get('isVirtual'):
raise ValidationException(
'You may not place items under a virtual folder.', field='folderId')
def testAccessFlags(self):
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(resp.json, {})
registerAccessFlag('my_key', name='hello', description='a custom flag')
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(resp.json, {
'my_key': {
'name': 'hello',
'description': 'a custom flag',
'admin': False
}
})
self.users[1] = User().load(self.users[1]['_id'], force=True)
user = self.users[1]
# Manage custom access flags on an access controlled resource
self.assertFalse(User().hasAccessFlags(user, user, flags=['my_key']))
# Admin should always have permission
self.assertTrue(User().hasAccessFlags(user, self.users[0], flags=['my_key']))
# Test the requireAccessFlags method
with self.assertRaises(AccessException):
User().requireAccessFlags(user, user=user, flags='my_key')
User().requireAccessFlags(user, user=self.users[0], flags='my_key')
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Test loadmodel requiredFlags argument via REST endpoint
resp = self.request(
'/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
self.assertStatus(resp, 403)
user = User().setAccessList(self.users[0], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'not a registered flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key']
}]
}, save=True)
resp = self.request(
'/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(resp.json, 'success')
# Only registered flags should be stored
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['my_key'])
self.assertTrue(User().hasAccessFlags(user, user, flags=['my_key']))
# Create an admin-only access flag
registerAccessFlag('admin_flag', name='admin flag', admin=True)
# Non-admin shouldn't be able to set it
user = User().setAccessList(self.users[0], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': []
}, save=True, user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Admin user should be able to set it
user = User().setAccessList(self.users[1], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
}, save=True, user=self.users[0])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['admin_flag'])
# An already-enabled admin-only flag should stay enabled for non-admin user
user = User().setAccessList(self.users[1], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
}, save=True, user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(set(acl['users'][0]['flags']), {'my_key', 'admin_flag'})
self.assertEqual(acl['groups'][0]['flags'], ['admin_flag'])
# Test setting public flags on a collection and folder
collectionModel = Collection()
folderModel = Folder()
itemModel = Item()
collection = collectionModel.createCollection('coll', creator=self.users[0], public=True)
folder = folderModel.createFolder(
collection, 'folder', parentType='collection', creator=self.users[0])
# Add an item to the folder so we can test AclMixin flag behavior
item = itemModel.createItem(folder=folder, name='test', creator=self.users[0])
folder = folderModel.setUserAccess(
folder, self.users[1], level=AccessType.ADMIN, save=True, currentUser=self.users[0])
with self.assertRaises(AccessException):
collectionModel.requireAccessFlags(collection, user=None, flags='my_key')
# Test AclMixin flag behavior
with self.assertRaises(AccessException):
itemModel.requireAccessFlags(item, user=None, flags='my_key')
self.assertFalse(itemModel.hasAccessFlags(item, user=None, flags='my_key'))
collection = collectionModel.setAccessList(
collection, access=collection['access'], save=True, recurse=True, user=self.users[0],
publicFlags=['my_key'])
collectionModel.requireAccessFlags(collection, user=None, flags='my_key')
# Make sure recursive setting of public flags worked
folder = folderModel.load(folder['_id'], force=True)
self.assertEqual(folder['publicFlags'], ['my_key'])
itemModel.requireAccessFlags(item, user=None, flags='my_key')
# Non-admin shouldn't be able to set admin-only public flags
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag'], user=self.users[1], save=True)
self.assertEqual(folder['publicFlags'], [])
# Admin users should be able to set admin-only public flags
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag'], user=self.users[0], save=True, append=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
# Non-admin users can set admin-only public flags if they are already enabled
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag', 'my_key'], user=self.users[1], save=True)
self.assertEqual(set(folder['publicFlags']), {'admin_flag', 'my_key'})
# Test "force" options
folder = folderModel.setPublicFlags(folder, flags='admin_flag', force=True, save=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
folder = folderModel.setAccessList(folder, access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'admin_flag']
}],
'groups': []
}, save=True, force=True)
folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')
folder = folderModel.setUserAccess(
folder, self.users[1], level=AccessType.READ, save=True, force=True, flags=[])
self.assertFalse(folderModel.hasAccessFlags(folder, self.users[1], flags='my_key'))
folder = folderModel.setGroupAccess(
folder, self.group, level=AccessType.READ, save=True, force=True, flags='my_key')
folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')
# Testing with flags=None should give sensible behavior
folderModel.requireAccessFlags(folder, user=None, flags=None)
# Test filtering results by access flags (both ACModel and AclMixin)
for model, doc in ((folderModel, folder), (itemModel, item)):
cursor = model.find({})
self.assertGreater(len(list(cursor)), 0)
cursor = model.find({})
filtered = list(model.filterResultsByPermission(
cursor, user=None, level=AccessType.READ, flags='my_key'))
self.assertEqual(len(filtered), 0)
cursor = model.find({})
filtered = list(model.filterResultsByPermission(
cursor, user=self.users[1], level=AccessType.READ, flags=('my_key', 'admin_flag')))
self.assertEqual(len(filtered), 1)
self.assertEqual(filtered[0]['_id'], doc['_id'])
def testFolderAccessAndDetails(self):
# create a folder to work with
folder = Folder().createFolder(
parent=self.admin, parentType='user', creator=self.admin,
name='Folder')
resp = self.request(
path='/folder/%s/access' % folder['_id'], method='GET',
user=self.admin)
self.assertStatusOk(resp)
access = resp.json
self.assertEqual(access, {
'users': [{
'login': self.admin['login'],
'level': AccessType.ADMIN,
'id': str(self.admin['_id']),
'flags': [],
'name': '%s %s' % (
self.admin['firstName'], self.admin['lastName'])}],
'groups': []
})
self.assertTrue(not folder.get('public'))
# Setting the access list with bad json should throw an error
resp = self.request(
path='/folder/%s/access' % folder['_id'], method='PUT',
user=self.admin, params={'access': 'badJSON'})
self.assertStatus(resp, 400)
# Change the access to public
resp = self.request(
path='/folder/%s/access' % folder['_id'], method='PUT',
user=self.admin,
params={'access': json.dumps(access), 'public': True})
self.assertStatusOk(resp)
resp = self.request(
path='/folder/%s' % folder['_id'], method='GET',
user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json['public'], True)
# Create an item in the folder
Item().createItem(folder=folder, creator=self.admin, name='Item')
# Create a public and private folder within the folder
Folder().createFolder(
parent=folder, parentType='folder', creator=self.admin,
name='Public', public=True)
Folder().createFolder(
parent=folder, parentType='folder', creator=self.admin,
name='Private', public=False)
# Test folder details as anonymous
resp = self.request(
path='/folder/%s/details' % str(folder['_id']))
self.assertStatusOk(resp)
self.assertEqual(resp.json['nItems'], 1)
self.assertEqual(resp.json['nFolders'], 1)
# Test folder details as admin
resp = self.request(
path='/folder/%s/details' % str(folder['_id']), user=self.admin)
self.assertStatusOk(resp)
self.assertEqual(resp.json['nItems'], 1)
self.assertEqual(resp.json['nFolders'], 2)
class HdfsAssetstoreResource(Resource):
def __init__(self):
super(HdfsAssetstoreResource, self).__init__()
self.resourceName = 'hdfs_assetstore'
self.route('PUT', (':id', 'import'), self.importData)
self.folderModel = Folder() # Save to avoid many lookups
self.itemModel = Item()
self.fileModel = File()
def _importFile(self, parent, name, user, assetstore, node):
item = self.itemModel.findOne({
'folderId': parent['_id'],
'name': name
})
if item is None:
item = self.itemModel.createItem(
name=name, creator=user, folder=parent)
file = self.fileModel.findOne({
'name': name,
'itemId': item['_id']
})
if file is None:
file = self.fileModel.createFile(
creator=user, item=item, name=name, size=node['length'],
assetstore=assetstore, mimeType=None, saveFile=False)
file['hdfs'] = {
'imported': True,
'path': node['path']
}
self.fileModel.save(file)
def _importData(self, parentType, parent, assetstore, client, path, ctx,
user):
for node in client.ls([path]):
ctx.update(message='Importing ' + node['path'])
name = posixpath.basename(node['path'])
if node['file_type'] == 'd':
folder = self.folderModel.findOne({
'parentId': parent['_id'],
'name': name,
'parentCollection': parentType
})
if folder is None:
folder = self.folderModel.createFolder(
parent, name, parentType=parentType, creator=user)
self._importData('folder', folder, assetstore, client,
node['path'], ctx, user)
elif node['file_type'] == 'f' and parentType == 'folder':
self._importFile(parent, name, user, assetstore, node)
@access.admin
@loadmodel(model='assetstore')
@describeRoute(
Description('Import a data hierarchy from an HDFS instance.')
.notes('Only site administrators may use this endpoint.')
.param('id', 'The ID of the assetstore representing the HDFS instance.',
paramType='path')
.param('parentId', 'The ID of the parent object in the Girder data '
'hierarchy under which to import the files.')
.param('parentType', 'The type of the parent object to import into.',
enum=('folder', 'user', 'collection'), required=False)
.param('path', 'Root of the directory structure (relative to the root '
'of the HDFS) to import.')
.param('progress', 'Whether to record progress on this operation ('
'default=False)', required=False, dataType='boolean')
.errorResponse()
.errorResponse('You are not an administrator.', 403)
)
def importData(self, assetstore, params):
self.requireParams(('parentId', 'path'), params)
user = self.getCurrentUser()
parentType = params.get('parentType', 'folder')
if parentType not in ('user', 'collection', 'folder'):
raise RestException('Invalid parentType.')
parent = self.model(parentType).load(params['parentId'], force=True, exc=True)
progress = self.boolParam('progress', params, default=False)
client = HdfsClient(
host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False)
path = params['path']
with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx:
try:
self._importData(parentType, parent, assetstore, client, path, ctx, user)
except FileNotFoundException:
raise RestException('File not found: %s.' % path)
class GeoParser(object):
@cherrypy.expose
def index(self):
tmpl = lookup.get_template("index.html")
return tmpl.render(salutation="Hello", target="World")
if __name__ == '__main__':
girderUser = User()
girderCollection = Collection()
girderFolder = Folder()
try:
currentUser = girderUser.getAdmins()[0]
except:
print "ERROR: No user registered or logged in."
existinig_collections = [each['name'] for each in girderCollection.list()]
if len(existinig_collections) == 0 or not GIRDER_COLLECTION in existinig_collections:
girderCollection.createCollection(GIRDER_COLLECTION, currentUser)
existinig_collections = [each['name'] for each in girderCollection.list()]
else:
print "ERROR: Cannot create Collection. User not registered or logged in."
try:
for collection in girderCollection.list():