Пример #1
0
def createNotebooks(event):
    user = event.info
    folder_model = Folder()

    result = lookUpPath('user/%s/Private' % user['login'], force=True)
    private_folder = result['document']

    oc_folder = folder_model.createFolder(private_folder, 'oc',
                                          parentType='folder',
                                          creator=user,
                                          public=True,
                                          reuseExisting=True)

    notebook_folder = folder_model.createFolder(oc_folder, 'notebooks',
                                                parentType='folder',
                                                creator=user,
                                                public=True,
                                                reuseExisting=True)

    notebooks_dir = os.path.join(os.path.dirname(__file__), 'notebooks')

    upload_model = Upload()
    for file in glob.glob('%s/*.ipynb' % notebooks_dir):
        size =  os.path.getsize(file)
        name = os.path.basename(file)
        with open(file, 'rb') as fp:
            upload_model.uploadFromFile(
                fp, size=size, name=name, parentType='folder',
                parent={'_id': ObjectId(notebook_folder['_id'])}, user=user,
                mimeType='application/x-ipynb+json')
Пример #2
0
    def testYAMLAliases(self):
        folderModel = Folder()
        aliasedFolders = list(folderModel.find({'name': 'Common'}, force=True))
        self.assertTrue(len(aliasedFolders) == 2)

        for folder in aliasedFolders:
            self.assertTrue(
                len(list(folderModel.childItems(folder, force=True))) == 2
            )
Пример #3
0
    def __init__(self):
        super(HdfsAssetstoreResource, self).__init__()
        self.resourceName = 'hdfs_assetstore'
        self.route('PUT', (':id', 'import'), self.importData)

        self.folderModel = Folder()  # Save to avoid many lookups
        self.itemModel = Item()
        self.fileModel = File()
Пример #4
0
 def stream():
     z = ziputil.ZipGenerator(folder['name'])
     for (path, file) in Folder().fileList(folder,
                                           user=user,
                                           subpath=False):
         for data in z.addFile(file, path):
             yield data
     for data in z.addFile(gen, "output_tracks.csv"):
         yield data
     yield z.footer()
Пример #5
0
    def test_mapping_creation(self):
        new_folder = Folder().createFolder(
            self.base_collection,
            random_string(),
            creator=self.users["sally"],
            parentType="collection",
            public=True,
            reuseExisting=True,
        )

        resp = self.request(
            path="/folder/{_id}".format(**new_folder),
            method="PUT",
            user=self.users["sally"],
            params={
                "fsPath": "/etc",
                "isMapping": True
            },  # H4ck3r detected!
        )
        self.assertStatus(resp, 403)
        self.assertEqual(
            resp.json,
            {
                "message": "Must be admin to setup virtual folders.",
                "type": "access"
            },
        )

        resp = self.request(
            path="/folder/{_id}".format(**new_folder),
            method="PUT",
            user=self.users["admin"],
            params={
                "fsPath": "/etc",
                "isMapping": True
            },  # G0d
        )
        self.assertStatusOk(resp)
        self.assertHasKeys(resp.json, ["fsPath", "isMapping"])
        self.assertEqual(resp.json["fsPath"], "/etc")
        self.assertEqual(resp.json["isMapping"], True)

        Folder().remove(new_folder)
Пример #6
0
def _validateFolder(event):
    doc = event.info

    if 'isVirtual' in doc and not isinstance(doc['isVirtual'], bool):
        raise ValidationException('The isVirtual field must be boolean.',
                                  field='isVirtual')

    if doc.get('isVirtual'):
        # Make sure it doesn't have children
        if list(Folder().childItems(doc, limit=1)):
            raise ValidationException(
                'Virtual folders may not contain child items.',
                field='isVirtual')
        if list(Folder().find(
            {
                'parentId': doc['_id'],
                'parentCollection': 'folder'
            }, limit=1)):
            raise ValidationException(
                'Virtual folders may not contain child folders.',
                field='isVirtual')
    if doc['parentCollection'] == 'folder':
        parent = Folder().load(event.info['parentId'], force=True, exc=True)
        if parent.get('isVirtual'):
            raise ValidationException(
                'You may not place folders under a virtual folder.',
                field='folderId')

    if 'virtualItemsQuery' in doc:
        try:
            json.loads(doc['virtualItemsQuery'])
        except (TypeError, ValueError):
            raise ValidationException(
                'The virtual items query must be valid JSON.',
                field='virtualItemsQuery')

    if 'virtualItemsSort' in doc:
        try:
            json.loads(doc['virtualItemsSort'])
        except (TypeError, ValueError):
            raise ValidationException(
                'The virtual items sort must be valid JSON.',
                field='virtualItemsSort')
Пример #7
0
def _virtualFolderDetails(self, event):
    folderId = event.info['id']
    user = self.getCurrentUser()
    folder = Folder().load(folderId, user=user, level=AccessType.READ)

    if not folder or not folder.get(
            'isVirtual') or 'virtualItemsQuery' not in folder:
        return  # Parent is not a virtual folder, proceed as normal

    q = json_util.loads(folder['virtualItemsQuery'])
    item = Item()
    # Virtual folders can't contain subfolders
    result = {
        'nFolders':
        0,
        'nItems':
        item.findWithPermissions(q, user=user, level=AccessType.READ).count()
    }
    event.preventDefault().addResponse(result)
Пример #8
0
 def ensure_version(self, event: events.Event):
     params = event.info.get("params", {})
     taleId = event.info.get("id")
     version_id = params.get("versionId")
     if not version_id:
         try:
             version = self.create(taleId=taleId,
                                   allowRename=True,
                                   params={})
             # Above obj is filtered model so we need to reload it...
             version = Folder().load(version["_id"], force=True)
         except RestException as exc:
             if exc.code == 303:
                 version = Folder().load(exc.extra, force=True)
             else:
                 raise
         # We're using 'updated' field to bump the version to the top of
         # Folder().list(). We're gonna update it either way later on.
         Folder().updateFolder(version)
Пример #9
0
def testWebClientAnnotationSpec(boundServer, fsAssetstore, db, admin):
    # Create an item in the Public folder
    publicFolder = next(
        folder for folder in Folder().childFolders(parent=admin, parentType='user', user=admin)
        if folder['public'] is True)
    Item().createItem('Empty', admin, publicFolder)

    spec = os.path.join(os.path.dirname(__file__), 'web_client_specs', 'annotationSpec.js')

    runWebClientTest(boundServer, spec, 15000)
Пример #10
0
    def updateFolderAccess(self, phase, submissions):
        """
        Synchronize access control between the phase and submission folders for
        the phase. Phase admins should have read access on the submission
        folders.
        """
        # Get phase admin users
        phaseAcl = Phase().getFullAccessList(phase)
        phaseAdminUserIds = set([user['id']
                                 for user in phaseAcl.get('users')
                                 if user['level'] >= AccessType.WRITE])
        phaseAdminUsers = [User().load(userId, force=True, exc=True)
                           for userId in phaseAdminUserIds]

        # Update submission folder ACL for current phase admins
        try:
            for sub in submissions:
                folder = Folder().load(sub['folderId'], force=True)
                if not folder:
                    continue
                folderAcl = Folder().getFullAccessList(folder)

                # Revoke access to users who are not phase admins; ignore folder
                # owner
                usersToRemove = [User().load(user['id'], force=True,
                                             exc=True)
                                 for user in folderAcl.get('users')
                                 if (user['id'] not in phaseAdminUserIds and
                                     user['id'] != folder['creatorId'])]
                for user in usersToRemove:
                    Folder().setUserAccess(folder, user, None)

                # Add access to phase admins; ignore folder owner
                usersToAdd = [user for user in phaseAdminUsers
                              if user['_id'] != folder['creatorId']]
                for user in usersToAdd:
                    Folder().setUserAccess(folder, user, AccessType.READ)

                # Save folder if access changed
                if usersToRemove or usersToAdd:
                    Folder().save(folder, validate=False)
        except TypeError:
            raise ValidationException('A list of submissions is required.')
Пример #11
0
    def create(
        self,
        tale: dict,
        name: Optional[str],
        versionsDir: Path,
        versionsRoot: dict,
        user=None,
        force=False,
    ) -> dict:
        last = self.getLastVersion(versionsRoot)
        last_restore = Folder().load(tale.get("restoredFrom", ObjectId()),
                                     force=True)
        workspace = Folder().load(tale["workspaceId"], force=True)
        crtWorkspace = Path(workspace["fsPath"])

        # NOTE: order is important, we want oldWorkspace -> last.workspace
        for version in (last_restore, last):
            oldWorkspace = (None if version is None else
                            Path(version["fsPath"]) / "workspace")
            if (not force and self.is_same(tale, version, user)
                    and self.sameTree(oldWorkspace, crtWorkspace)):
                assert version is not None
                raise RestException("Not modified",
                                    code=303,
                                    extra=str(version["_id"]))

        new_version = self.createSubdir(versionsDir,
                                        versionsRoot,
                                        name,
                                        user=user)

        try:
            self.snapshot(last, tale, new_version, user=user, force=force)
            return new_version
        except Exception:  # NOQA
            try:
                shutil.rmtree(new_version["fsPath"])
                Folder().remove(new_version)
            except Exception as ex:  # NOQA
                logger.warning(
                    "Exception caught while rolling back version ckeckpoint.",
                    ex)
            raise
Пример #12
0
 def tearDown(self):
     for folder in (self.public_folder, self.private_folder,
                    self.regular_folder):
         Folder().remove(folder)
     Collection().remove(self.base_collection)
     for user in self.users.values():
         User().remove(user)
     for root in (self.public_root, self.private_root):
         shutil.rmtree(root)
     super(ItemOperationsTestCase, self).tearDown()
Пример #13
0
    def getImageList(self, folderId, limit, offset):
        printOk('getImageList() was called!')
        limit, offset = int(limit), int(offset)
        self.user = self.getCurrentUser()
        folder = Folder().load(folderId,
                               level=AccessType.READ,
                               user=self.getCurrentUser())
        items = Folder().childItems(folder, limit=limit, offset=offset)
        items = self.__filter(items,
                              exts=[".jpg", ".jpeg", ".png", ".PNG", ".svs"])
        ret_files = []
        for item in items:
            # TODO: remove this function
            filename = os.path.splitext(item['name'])[0]
            printOk("filename: " + filename)
            ret_files.append(item)

        cherrypy.response.headers["Content-Type"] = "application/json"
        return dumps(ret_files)
Пример #14
0
    def testTaleCopy(self):
        from girder.plugins.wholetale.models.tale import Tale
        from girder.plugins.wholetale.constants import TaleStatus
        from girder.plugins.jobs.models.job import Job
        from girder.plugins.jobs.constants import JobStatus
        tale = Tale().createTale(self.image, [],
                                 creator=self.admin,
                                 public=True)
        workspace = Tale().createWorkspace(tale)
        # Below workarounds a bug, it will be addressed elsewhere.
        workspace = Folder().setPublic(workspace, True, save=True)

        adapter = assetstore_utilities.getAssetstoreAdapter(self.ws_assetstore)
        size = 101
        data = BytesIO(b' ' * size)
        files = []
        files.append(Upload().uploadFromFile(data,
                                             size,
                                             'file01.txt',
                                             parentType='folder',
                                             parent=workspace,
                                             assetstore=self.ws_assetstore))
        fullPath = adapter.fullPath(files[0])

        # Create a copy
        resp = self.request(path='/tale/{_id}/copy'.format(**tale),
                            method='POST',
                            user=self.user)
        self.assertStatusOk(resp)

        new_tale = resp.json
        self.assertFalse(new_tale['public'])
        self.assertEqual(new_tale['dataSet'], tale['dataSet'])
        self.assertEqual(new_tale['copyOfTale'], str(tale['_id']))
        self.assertEqual(new_tale['imageId'], str(tale['imageId']))
        self.assertEqual(new_tale['creatorId'], str(self.user['_id']))
        self.assertEqual(new_tale['status'], TaleStatus.PREPARING)

        copied_file_path = re.sub(workspace['name'], new_tale['_id'], fullPath)
        job = Job().findOne({'type': 'wholetale.copy_workspace'})
        for i in range(10):
            job = Job().load(job['_id'], force=True)
            if job['status'] == JobStatus.SUCCESS:
                break
            time.sleep(0.1)
        self.assertTrue(os.path.isfile(copied_file_path))
        resp = self.request(path='/tale/{_id}'.format(**new_tale),
                            method='GET',
                            user=self.user)
        self.assertStatusOk(resp)
        new_tale = resp.json
        self.assertEqual(new_tale['status'], TaleStatus.READY)

        Tale().remove(new_tale)
        Tale().remove(tale)
Пример #15
0
 def filesResponse(self, path):
     user = self.getCurrentUser()
     path = '/'.join(path)
     # Handle /collection and /user specially
     if path in ('user', 'collection'):
         if not cherrypy.request.path_info.endswith('/'):
             return self._forward()
         model = User() if path == 'user' else Collection()
         return self._listDirectory(path, model.list(
             user=user, sort=[('user' if path == 'user' else 'name', SortDir.ASCENDING)]))
     resource = path_util.lookUpPath(path, user)['document']
     singleFile = None
     if resource['_modelType'] == 'file':
         singleFile = resource
     elif resource['_modelType'] == 'item':
         singleFile = None
         for _path, file in Item().fileList(doc=resource, user=user, subpath=True, data=False):
             if singleFile is None:
                 singleFile = file
             else:
                 singleFile = False
                 break
     if singleFile is not False and singleFile is not None:
         offset, endByte = 0, None
         rangeHeader = cherrypy.lib.httputil.get_ranges(
             cherrypy.request.headers.get('Range'), singleFile.get('size', 0))
         if rangeHeader and len(rangeHeader):
             offset, endByte = rangeHeader[0]
         singleFile = File().load(singleFile['_id'], user=user, level=AccessType.READ)
         return File().download(singleFile, offset, endByte=endByte)
     if not cherrypy.request.path_info.endswith('/'):
         return self._forward()
     children = []
     if resource['_modelType'] != 'item':
         children.extend(Folder().childFolders(
             parentType=resource['_modelType'], parent=resource, user=user,
             sort=[('name', SortDir.ASCENDING)]))
     if resource['_modelType'] == 'folder':
         children.extend(Folder().childItems(resource, sort=[('name', SortDir.ASCENDING)]))
     if resource['_modelType'] == 'item':
         children.extend(Item().childFiles(resource, sort=[('name', SortDir.ASCENDING)]))
     return self._listDirectory(path, children)
Пример #16
0
    def testVirtualFolderValidation(self):
        # Can't make folder virtual if it has children
        subfolder = Folder().createFolder(self.f1, 'sub', creator=self.admin)
        self.f1['isVirtual'] = True

        with six.assertRaisesRegex(
                self, ValidationException,
                'Virtual folders may not contain child folders.'):
            Folder().save(self.f1)

        Folder().remove(subfolder)
        item = Item().createItem('i', creator=self.admin, folder=self.f1)

        with six.assertRaisesRegex(
                self, ValidationException,
                'Virtual folders may not contain child items.'):
            Folder().save(self.f1)

        Item().remove(item)
        Folder().save(self.f1)

        # Can't make subfolders or items under a virtual folder
        with six.assertRaisesRegex(
                self, ValidationException,
                'You may not place items under a virtual folder.'):
            Item().createItem('i', creator=self.admin, folder=self.f1)

        with six.assertRaisesRegex(
                self, ValidationException,
                'You may not place folders under a virtual folder.'):
            Folder().createFolder(self.f1, 'f', creator=self.admin)

        # Can't move an item under a virtual folder
        item = Item().createItem('i', creator=self.admin, folder=self.f2)
        with six.assertRaisesRegex(
                self, ValidationException,
                'You may not place items under a virtual folder.'):
            Item().move(item, self.f1)

        # Ensure JSON for query
        self.f1['virtualItemsQuery'] = 'not JSON'
        with six.assertRaisesRegex(
                self, ValidationException,
                'The virtual items query must be valid JSON.'):
            Folder().save(self.f1)

        del self.f1['virtualItemsQuery']
        self.f1['virtualItemsSort'] = 'not JSON'
        with six.assertRaisesRegex(
                self, ValidationException,
                'The virtual items sort must be valid JSON.'):
            Folder().save(self.f1)
Пример #17
0
 def remove(self, calc, user=None, force=False):
     super(Calculation, self).remove(calc)
     # remove ingested file
     file_id = calc.get('fileId')
     if file_id is not None:
         file = File().load(file_id, user=user, level=AccessType.WRITE)
         if file:
             item = Item().load(file['itemId'],
                                user=user,
                                level=AccessType.WRITE)
             if item:
                 Item().remove(item)
     # remove scratch folder with calculation output
     scratch_folder_id = calc.get('scratchFolderId')
     if scratch_folder_id is not None:
         folder = Folder().load(scratch_folder_id,
                                user=user,
                                level=AccessType.WRITE)
         if folder:
             Folder().remove(folder)
Пример #18
0
def childItems(self, folder, limit=0, offset=0, sort=None, filters=None,
               includeVirtual=False, **kwargs):
    if not includeVirtual or not folder.get('isVirtual') or 'virtualItemsQuery' not in folder:
        return Folder._childItemsBeforeHUI(
            self, folder, limit=limit, offset=offset, sort=sort,
            filters=filters, **kwargs)
    q = json_util.loads(folder['virtualItemsQuery'])
    if 'virtualItemsSort' in folder and sort is None:
        sort = json.loads(folder['virtualItemsSort'])
    q.update(filters or {})
    return Item().find(q, limit=limit, offset=offset, sort=sort, **kwargs)
Пример #19
0
def saveImportAttributes(folder, attributes, user):
    attributes_dict = fromMeta(folder, 'attributes', {})
    # we don't overwrite any existing meta attributes
    for attribute in attributes.values():
        validated: models.Attribute = models.Attribute(**attribute)
        if attribute['key'] not in attributes_dict:
            attributes_dict[str(
                validated.key)] = validated.dict(exclude_none=True)

    folder['meta']['attributes'] = attributes_dict
    Folder().save(folder)
Пример #20
0
def findSharedDatasetFolders(currentUser):
    folderModel = Folder()
    groupModel = ModelImporter.model('group')
    datasetSharingGroup = groupModel.findOne(
        query={'name': PluginSettings.DATASET_SHARING_GROUP_NAME})
    if not datasetSharingGroup:
        raise AccessException('user group "{0}" doesn\'t exist'.format(
            PluginSettings.DATASET_SHARING_GROUP_NAME))
    if datasetSharingGroup['_id'] not in currentUser['groups']:
        raise AccessException(
            'user doesn\'t belong to user group "{0}"'.format(
                PluginSettings.DATASET_SHARING_GROUP_NAME))

    folders = folderModel.find({
        'baseParentType': 'user',
        'parentCollection': 'user',
        'access.groups.id': datasetSharingGroup['_id'],
        'name': PluginSettings.MINERVA_SHARED_DATASET
    })
    return folders
Пример #21
0
 def create_folder(self, event, path, root, user=None):
     params = event.info["params"]
     new_path = path / params["name"]
     try:
         new_path.mkdir()
     except FileExistsError:
         raise ValidationException(
             "A folder with that name already exists here.", "name")
     event.preventDefault().addResponse(Folder().filter(self.vFolder(
         new_path, root),
                                                        user=user))
Пример #22
0
def training_output_folder(user: types.GirderUserModel) -> types.GirderModel:
    """Ensure that the user has a training results folder."""
    viameFolder = Folder().createFolder(
        user,
        constants.ViameDataFolderName,
        description="VIAME data storage.",
        parentType="user",
        public=False,
        creator=user,
        reuseExisting=True,
    )

    return Folder().createFolder(
        viameFolder,
        constants.TrainingOutputFolderName,
        description="Results from VIAME model training are placed here.",
        public=False,
        creator=user,
        reuseExisting=True,
    )
Пример #23
0
    def testPrivateUser(self):
        """
        Make sure private users behave correctly.
        """
        # Create an admin user
        User().createUser(firstName='Admin',
                          lastName='Admin',
                          login='******',
                          email='*****@*****.**',
                          password='******')

        # Register a private user (non-admin)
        pvt = User().createUser(firstName='Guy',
                                lastName='Noir',
                                login='******',
                                email='*****@*****.**',
                                password='******',
                                public=False)

        self.assertEqual(pvt['public'], False)

        folder = six.next(Folder().childFolders(parentType='user', parent=pvt))

        # Private users should be able to upload files
        resp = self.request(path='/item',
                            method='POST',
                            user=pvt,
                            params={
                                'name': 'foo.txt',
                                'folderId': folder['_id']
                            })
        self.assertStatusOk(resp)
        itemId = resp.json['_id']

        resp = self.request(path='/file',
                            method='POST',
                            user=pvt,
                            params={
                                'parentType': 'item',
                                'parentId': itemId,
                                'name': 'foo.txt',
                                'size': 5,
                                'mimeType': 'text/plain'
                            })
        self.assertStatusOk(resp)

        fields = [('offset', 0), ('uploadId', resp.json['_id'])]
        files = [('chunk', 'foo.txt', 'hello')]
        resp = self.multipartRequest(path='/file/chunk',
                                     user=pvt,
                                     fields=fields,
                                     files=files)
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['itemId'], itemId)
Пример #24
0
def _folderUpdate(self, event):
    params = event.info['params']
    if {'isVirtual', 'virtualItemsQuery', 'virtualItemsSort'} & set(params):
        folder = Folder().load(event.info['returnVal']['_id'], force=True)
        update = False

        if params.get('isVirtual') is not None:
            update = True
            folder['isVirtual'] = params['isVirtual']
        if params.get('virtualItemsQuery') is not None:
            update = True
            folder['virtualItemsQuery'] = params['virtualItemsQuery']
        if params.get('virtualItemsSort') is not None:
            update = True
            folder['virtualItemsSort'] = params['virtualItemsSort']

        if update:
            self.requireAdmin(self.getCurrentUser(), 'Must be admin to setup virtual folders.')
            folder = Folder().filter(Folder().save(folder), rest.getCurrentUser())
            event.preventDefault().addResponse(folder)
Пример #25
0
 def setStatus(self, rfolder: dict, status: Union[int, RunState]) -> None:
     # TODO: add heartbeats (runs must regularly update status, otherwise they are considered
     # failed)
     if isinstance(status, int):
         _status = RunState.ALL[status]
     else:
         _status = status
     rfolder[FIELD_STATUS_CODE] = _status.code
     Folder().save(rfolder)
     runDir = Path(rfolder["fsPath"])
     self.write_status(runDir, _status)
Пример #26
0
def drafts_collection(db):
    red_herring_collection = Collection().createCollection(
        "red_herring_collection", reuseExisting=True)
    # A folder that matches dandiset metadata, but not in drafts collection.
    red_herring_dandiset_000001_folder = Folder().createFolder(
        parent=red_herring_collection,
        parentType="collection",
        name="000001",
        public=True,
    )
    meta = {
        "dandiset": {
            "name": "red",
            "description": "herring",
            "identifier": "000001"
        }
    }
    Folder().setMetadata(red_herring_dandiset_000001_folder, meta)

    return get_or_create_drafts_collection()
Пример #27
0
def check_existing_annotations(event):
    info = event.info

    if "annotations.csv" in info["importPath"]:
        item = Item().findOne({"_id": info["id"]})
        item["meta"].update({
            "folderId": str(item["folderId"]),
            "pipeline": None
        })
        Item().save(item)

        folder = Folder().findOne({"_id": item["folderId"]})

        # FPS is hardcoded for now
        folder["meta"].update({
            "type": "image-sequence",
            "viame": True,
            "fps": 30
        })
        Folder().save(folder)
Пример #28
0
def parentChain(admin):
    # Create the parent chain
    F1 = Folder().createFolder(
        parent=admin, parentType='user', creator=admin,
        name='F1', public=True)
    F2 = Folder().createFolder(
        parent=F1, parentType='folder', creator=admin,
        name='F2', public=True)
    privateFolder = Folder().createFolder(
        parent=F2, parentType='folder', creator=admin,
        name='F3', public=False)
    F4 = Folder().createFolder(
        parent=privateFolder, parentType='folder', creator=admin,
        name='F4', public=True)
    yield {
        'folder1': F1,
        'folder2': F2,
        'privateFolder': privateFolder,
        'folder4': F4
    }
Пример #29
0
def getOrCreateRootFolder(name):
    collection = Collection().createCollection(
        name, public=True, reuseExisting=True)
    # For backward compat
    if not collection['public']:
        collection = Collection().save(
            Collection().setPublic(collection, True)
        )
    folder = Folder().createFolder(
        collection, name, parentType='collection', public=True, reuseExisting=True)
    return folder
Пример #30
0
    def _testUploadWithPath(self):
        testUser = User().createUser(firstName='Jeffrey',
                                     lastName='Abrams',
                                     login='******',
                                     password='******',
                                     email='*****@*****.**')
        publicFolder = six.next(Folder().childFolders(parentType='user',
                                                      parent=testUser,
                                                      user=None,
                                                      limit=1))
        self.client.upload(self.libTestDir, '/user/jjabrams/Public')

        parent = six.next(Folder().childFolders(parentType='folder',
                                                parent=publicFolder,
                                                user=testUser,
                                                limit=0))
        self.assertEqual([
            f['name'] for f in Folder().childFolders(
                parentType='folder', parent=parent, user=testUser, limit=0)
        ], ['sub0', 'sub1', 'sub2'])
Пример #31
0
def import_recursive(job):
    try:
        root = job['kwargs']['root']
        token = job['kwargs']['token']

        user = User().load(job['userId'], force=True)

        children = list(Folder().childFolders(root, 'collection', user=user))
        count = len(children)
        progress = 0

        job = Job().updateJob(job,
                              log='Started TCGA import\n',
                              status=JobStatus.RUNNING,
                              progressCurrent=progress,
                              progressTotal=count)
        logger.info('Starting recursive TCGA import')

        for child in children:
            progress += 1
            try:
                msg = 'Importing "%s"' % child.get('name', '')
                job = Job().updateJob(job,
                                      log=msg,
                                      progressMessage=msg + '\n',
                                      progressCurrent=progress)
                logger.debug(msg)
                Cohort().importDocument(child,
                                        recurse=True,
                                        user=user,
                                        token=token,
                                        job=job)
                job = Job().load(id=job['_id'], force=True)

                # handle any request to stop execution
                if (not job or job['status']
                        in (JobStatus.CANCELED, JobStatus.ERROR)):
                    logger.info('TCGA import job halted with')
                    return

            except ValidationException:
                logger.warning('Failed to import %s' % child.get('name', ''))

        logger.info('Starting recursive TCGA import')
        job = Job().updateJob(job,
                              log='Finished TCGA import\n',
                              status=JobStatus.SUCCESS,
                              progressCurrent=count,
                              progressMessage='Finished TCGA import')
    except Exception as e:
        logger.exception('Importing TCGA failed with %s' % str(e))
        job = Job().updateJob(job,
                              log='Import failed with %s\n' % str(e),
                              status=JobStatus.ERROR)
Пример #32
0
 def _pruneOrphans(self, progress):
     count = 0
     models = [File(), Folder(), Item()]
     steps = sum(model.find().count() for model in models)
     progress.update(total=steps, current=0)
     for model in models:
         for doc in model.find():
             progress.update(increment=1)
             if model.isOrphan(doc):
                 model.remove(doc)
                 count += 1
     return count
Пример #33
0
    def processNotification(store: AssetstoreModel, rootFolder: GirderModel,
                            importPath: str):
        """
        Import at proper location
        """
        if rootFolder is None:
            raise RestException('Root folder missing')
        # Find the correct import location given rootFolder and importPath
        owner = User().findOne({
            '_id':
            ObjectId(rootFolder['creatorId'] or rootFolder['baseParentId'])
        })
        if owner is None:
            raise RestException('Root has no owner')

        target = rootFolder

        importPath = importPath.lstrip('/')
        realImportPathDir, _ = os.path.split(importPath)
        storePrefix = str(store['prefix']).lstrip('/')
        common_path = os.path.commonpath([storePrefix, importPath])

        realImportPath = common_path
        path_from_root = importPath[len(common_path):].lstrip('/')
        path_from_root_list, _ = os.path.split(path_from_root)

        for folder_name in path_from_root_list.split('/'):
            new_target = Folder().findOne({
                'parentId': ObjectId(target['_id']),
                'name': folder_name,
            })
            new_import_path = f'{realImportPath}/{folder_name}'
            if new_target is not None:
                target = new_target
                realImportPath = new_import_path
            else:
                break

        realImportPath = realImportPath.lstrip('/')

        if realImportPath == realImportPathDir:
            # All the chain of parent directories exist
            realImportPath = importPath

        Assetstore().importData(
            store,
            target,
            'folder',
            {'importPath': realImportPath},
            None,
            owner,
            force_recursive=False,
        )
Пример #34
0
    def __init__(self):
        super().__init__()

        self.coll_m = Collection()
        self.file_m = File()
        self.folder_m = Folder()
        self.item_m = Item()
        self.upload_m = Upload()
        self.asset_m = Assetstore()
        self.user_m = User()

        self.setupRoutes()
Пример #35
0
    def importSlide(self, params):
        user = self.getCurrentUser()
        token = self.getCurrentToken()
        self.requireParams('folderId', params)

        folder = Folder().load(id=params['folderId'],
                               user=user,
                               level=AccessType.WRITE,
                               exc=True)

        slide = Slide().importDocument(folder, user=user, token=token)
        return slide
Пример #36
0
def _virtualChildItems(self, event):
    params = event.info['params']

    if 'folderId' not in params:
        return  # This is not a child listing request

    user = self.getCurrentUser()
    folder = Folder().load(params['folderId'], user=user, level=AccessType.READ)

    if not folder.get('isVirtual') or 'virtualItemsQuery' not in folder:
        return  # Parent is not a virtual folder, proceed as normal

    limit, offset, sort = self.getPagingParameters(params, defaultSortField='name')
    q = json_util.loads(folder['virtualItemsQuery'])

    if 'virtualItemsSort' in folder:
        sort = json.loads(folder['virtualItemsSort'])

    item = Item()
    # These items may reside in folders that the user cannot read, so we must filter
    items = item.filterResultsByPermission(
        item.find(q, sort=sort), user, level=AccessType.READ, limit=limit, offset=offset)
    event.preventDefault().addResponse([item.filter(i, user) for i in items])
Пример #37
0
def _validateFolder(event):
    doc = event.info

    if 'isVirtual' in doc and not isinstance(doc['isVirtual'], bool):
        raise ValidationException('The isVirtual field must be boolean.', field='isVirtual')

    if doc.get('isVirtual'):
        # Make sure it doesn't have children
        if list(Folder().childItems(doc, limit=1)):
            raise ValidationException(
                'Virtual folders may not contain child items.', field='isVirtual')
        if list(Folder().find({
            'parentId': doc['_id'],
            'parentCollection': 'folder'
        }, limit=1)):
            raise ValidationException(
                'Virtual folders may not contain child folders.', field='isVirtual')
    if doc['parentCollection'] == 'folder':
        parent = Folder().load(event.info['parentId'], force=True, exc=True)
        if parent.get('isVirtual'):
            raise ValidationException(
                'You may not place folders under a virtual folder.', field='folderId')

    if 'virtualItemsQuery' in doc:
        try:
            json.loads(doc['virtualItemsQuery'])
        except (TypeError, ValueError):
            raise ValidationException(
                'The virtual items query must be valid JSON.', field='virtualItemsQuery')

    if 'virtualItemsSort' in doc:
        try:
            json.loads(doc['virtualItemsSort'])
        except (TypeError, ValueError):
            raise ValidationException(
                'The virtual items sort must be valid JSON.', field='virtualItemsSort')
Пример #38
0
 def __init__(self):
     super(Folder, self).__init__()
     self.resourceName = 'folder'
     self._model = FolderModel()
     self.route('DELETE', (':id',), self.deleteFolder)
     self.route('DELETE', (':id', 'contents'), self.deleteContents)
     self.route('GET', (), self.find)
     self.route('GET', (':id',), self.getFolder)
     self.route('GET', (':id', 'details'), self.getFolderDetails)
     self.route('GET', (':id', 'access'), self.getFolderAccess)
     self.route('GET', (':id', 'download'), self.downloadFolder)
     self.route('GET', (':id', 'rootpath'), self.rootpath)
     self.route('POST', (), self.createFolder)
     self.route('PUT', (':id',), self.updateFolder)
     self.route('PUT', (':id', 'access'), self.updateFolderAccess)
     self.route('POST', (':id', 'copy'), self.copyFolder)
     self.route('PUT', (':id', 'metadata'), self.setMetadata)
     self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)
Пример #39
0
class AssignmentResource(Resource):
    def __init__(self):
        super().__init__()

        self.coll_m = Collection()
        self.file_m = File()
        self.folder_m = Folder()
        self.item_m = Item()
        self.upload_m = Upload()
        self.asset_m = Assetstore()
        self.user_m = User()

        self.setupRoutes()

    def setupRoutes(self):
        self.route('GET', (), handler=self.list)
        self.route('GET', (':a_id',), handler=self.getAssignment)
        self.route('GET', ('admin_data',), handler=self.getAdminData)

    def __findName(self, folder):
        imageFolder = self.__findImageFolder(folder)
        if isinstance(imageFolder, dict):
            return imageFolder['name']

        return ""

    def __findImageFolder(self, folder):
        owner = self.__findOwner(folder)
        ret = ""
        if self.folder_m.getAccessLevel(folder, self.getCurrentUser()) == AccessType.ADMIN:
            # this folder was created by this user, and so it will contain the images
            # printFail(folder)
            ret = folder
        else:
            meta = folder['meta']
            # this is the label file, and so should only have one entry in the metadata
            assert len(meta) == 1
            # that one entry contains link to the image folder, key must be the creator of this folder
            assert str(owner['_id']) in meta, (str(owner['_id']), meta)

            ret = self.folder_m.load(meta[str(owner['_id'])],
                                     level=AccessType.READ,
                                     user=self.getCurrentUser())

        return ret

    def __findLabelFolder(self, folder):
        owner = self.__findOwner(folder)
        ret = []
        if self.folder_m.getAccessLevel(folder, self.getCurrentUser()) == AccessType.ADMIN:
            # this folder was created by this user, so it will contain images
            if 'meta' not in folder:
                return []

            meta = folder['meta']
            for k, v in meta.items():
                ret.append(self.folder_m.load(v, level=AccessType.READ, user=self.getCurrentUser()))
        else:
            # this folder was not created by this user, so it will contain labels
            ret = [folder]

        return ret

    def __findOwner(self, folder):
        aclList = Folder().getFullAccessList(folder)
        for acl in aclList['users']:
            if acl['level'] == AccessType.ADMIN:
                return self.user_m.load(str(acl['id']), level=AccessType.READ, user=self.getCurrentUser())
        return None

    @access.public
    @autoDescribeRoute(
        Description('Get list of all assignments that it owns or is a part of')
            .param('limit', 'Number of assignments to return')
            .param('offset', 'offset from 0th assignment to start looking for assignments'))
    @rest.rawResponse
    def list(self, limit, offset):
        try:
            printOk((limit, offset))
            ret = self.__list(int(limit), int(offset))
            cherrypy.response.headers["Content-Type"] = "application/json"
            return dumps(ret)

        except:
            printFail(traceback.print_exc)

    def __list(self, limit=0, offset=0):
        user = self.getCurrentUser()
        folders = self.folder_m.find({'parentId': user['_id'],
                                      'parentCollection': 'user'}, limit=limit, offset=offset)
        ret = []
        for folder in folders:
            if self.__findName(folder):
                ret.append({'name': self.__findName(folder),
                            'image_folder': self.__findImageFolder(folder),
                            'label_folders': self.__findLabelFolder(folder),
                            'owner': self.__findOwner(folder)})

        return ret

    def __getAssignment(self, _id):
        assignments = self.__list()
        for assignment in assignments:
            # printOk2(assignment)
            if str(assignment['image_folder']['_id']) == _id:
                return assignment

        return None

    def __getAnnotators(self, assignment):
        ret = []
        for label_folder in assignment['label_folders']:
            printOk(label_folder)
            ret.append({
                'user': self.user_m.load(label_folder['parentId'], user=self.getCurrentUser(), level=AccessType.READ),
                'expanded': False,
            })

        return ret

    @access.public
    @autoDescribeRoute(
        Description('Get assignment by id').param('a_id', 'folder id that controls the assignment'))
    @rest.rawResponse
    def getAssignment(self, a_id):
        try:
            assignment = self.__getAssignment(a_id)
            return dumps(assignment)
        except:
            printFail(traceback.print_exc)

    @access.public
    @autoDescribeRoute(
        Description('Get admin data associated with assignment id').param('a_id',
                                                                          'folder id that controls the assignment'))
    @rest.rawResponse
    def getAdminData(self, a_id):
        try:
            assignment = self.__getAssignment(a_id)
            if assignment['owner']['_id'] == self.getCurrentUser()['_id']:
                # then current user is this assignment's admin
                return dumps({'annotators': self.__getAnnotators(assignment)})

        except:
            printFail(traceback.print_exc)
Пример #40
0
class LabelResource(Resource):

    def __init__(self):
        super().__init__()
        self.resourceName = 'label'

        self.coll_m = Collection()
        self.file_m = File()
        self.folder_m = Folder()
        self.item_m = Item()
        self.upload_m = Upload()
        self.asset_m = Assetstore()

        self.setupRoutes()

    def setupRoutes(self):
        self.route('GET', (), handler=self.getLabelList)
        self.route('GET', (':label_id',), self.getLabel)
        self.route('GET', ('meta',), self.getLabelMeta)
        self.route('GET', ('create',), self.createLabelFile)
        self.route('GET', ('by_name',), self.getLabelByName)
        self.route('POST', (), self.postLabel)

    def createNewFile(self, folder, file_name):
        item = self.item_m.createItem(file_name,
                                      creator=self.getCurrentUser(),
                                      folder=folder,
                                      description='label file',
                                      reuseExisting=False)

        file = self.file_m.createFile(size=0,
                                      item=item,
                                      name=file_name,
                                      creator=self.getCurrentUser(),
                                      assetstore=self.asset_m.getCurrent(),
                                      mimeType="application/json")
        return file

    def copy(self, srcFile, destFile):
        upload = self.upload_m.createUploadToFile(destFile, self.getCurrentUser(), srcFile['size'])
        self.upload_m.handleChunk(upload=upload,
                                  chunk=RequestBodyStream(self.file_m.open(srcFile), size=destFile['size']),
                                  user=self.getCurrentUser())
        return upload

    @access.public
    @autoDescribeRoute(
        Description('Get label list'))
    @rest.rawResponse
    def getLabelList(self):
        printOk('getLabelsList() was called!')

        try:
            collection = list(self.coll_m.list(user=self.getCurrentUser(), offset=0, limit=1))[0]
            files = self.coll_m.fileList(collection, user=self.getCurrentUser(), data=False,
                                         includeMetadata=True, mimeFilter=['application/json'])
            files = list(files)
            cherrypy.response.headers["Content-Type"] = "application/json"
            return dumps(files)

        except:
            printFail(traceback.print_exc)

    @staticmethod
    def getOwnerId(folder):
        aclList = Folder().getFullAccessList(folder)
        for acl in aclList['users']:
            if acl['level'] == AccessType.ADMIN:
                return str(acl['id'])
        return None

    def getConfigFolder(self, label_folder_id):
        label_folder = Folder().load(label_folder_id,
                                     user=self.getCurrentUser(),
                                     level=AccessType.READ)
        ownerId = self.getOwnerId(label_folder)
        config_folder = self.folder_m.load(label_folder['meta'][ownerId], level=AccessType.READ,
                                           user=self.getCurrentUser())
        return config_folder

    def findConfig(self, folder_id):
        folder = self.getConfigFolder(folder_id)
        printOk2("Config folder {}".format(folder))
        files = self.folder_m.fileList(folder, self.getCurrentUser(), data=False)
        for file_path, file in files:
            printOk(file)
            if file['name'] == "config.json":
                return file

    def __findFile(self, folder, file_name):
        item = list(self.item_m.find({'folderId': folder['_id'], 'name': file_name}).limit(1))
        if not item:
            return None

        item = item[0]
        file = list(self.file_m.find({'itemId': item['_id']}).limit(1))

        if not file:
            return None

        return file[0]

    @access.public
    @autoDescribeRoute(
        Description('Create a new label file if it doesnt exist')
            .param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
    @rest.rawResponse
    def createLabelFile(self, file_name, folder_id):
        try:
            folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.WRITE)
            file = self.__findFile(folder, file_name)
            if not file:
                file = self.createNewFile(folder, file_name)
                config_file = self.findConfig(folder_id)
                if not config_file:
                    printFail("No config file found")
                    return errorMessage("No config file found")
                else:
                    res = self.copy(config_file, file)
                    return dumps({
                        "label_id": res['fileId']
                    })

            return dumps({
                "label_id": file['_id']
            })
        except:
            printFail(traceback.print_exc)
            cherrypy.response.status = 500

    @access.public
    @autoDescribeRoute(
        Description('Get labels by file_name')
            .param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
    @rest.rawResponse
    def getLabelByName(self, file_name, folder_id):
        try:
            folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.READ)
            file = self.__findFile(folder, file_name)
            cherrypy.response.headers["Content-Type"] = "application/json"
            if file:
                return self.file_m.download(file)
            else:
                return dumps({})
        except:
            printFail(traceback.print_exc)
            cherrypy.response.status = 500

    @access.public
    @autoDescribeRoute(
        Description('Get label by id')
            .param('label_id', 'label file id'))
    @rest.rawResponse
    def getLabel(self, label_id):
        try:
            file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
            printOk2(file)
            cherrypy.response.headers["Content-Type"] = "application/json"
            return self.file_m.download(file)
        except:
            # Unknown slug
            printFail(traceback.print_exc)
            cherrypy.response.status = 404

    @access.public
    @autoDescribeRoute(
        Description('Get label by id')
            .param('label_id', 'label file id'))
    def getLabelMeta(self, label_id):
        try:
            file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
            cherrypy.response.headers["Content-Type"] = "application/json"
            return dumps(file)
        except:
            # Unknown slug
            printFail(traceback.print_exc)
            cherrypy.response.status = 404

    @access.public
    @autoDescribeRoute(
        Description('Post label by id')
            .param('label_id', 'label file id'))
    @rest.rawResponse
    def postLabel(self, label_id, params):
        try:
            file = self.file_m.load(label_id, level=AccessType.WRITE, user=self.getCurrentUser())
            cherrypy.response.headers["Content-Type"] = "application/json"
            params['labels'] = json.loads(params['labels'])
            data = json.dumps(params, indent=2, sort_keys=True)
            upload = writeData(self.getCurrentUser(), file, data)
            printOk2(file)
            printOk(upload)
            return dumps(upload)

        except:
            # Unknown slug
            printFail(traceback.print_exc)
            cherrypy.response.status = 404

    @access.public
    @autoDescribeRoute(
        Description('Post label by id')
            .param('label_id', 'label file id'))
    @rest.rawResponse
    def strokeToOutline(self, strokes):
        pass
Пример #41
0
    def setCuration(self, folder, params):
        user = self.getCurrentUser()
        if CURATION not in folder:
            folder[CURATION] = dict(DEFAULTS)
        curation = folder[CURATION]
        oldCuration = dict(curation)

        # update enabled
        oldEnabled = curation.get(ENABLED, False)
        if ENABLED in params:
            self.requireAdmin(user)
            curation[ENABLED] = self.boolParam(ENABLED, params)
        enabled = curation.get(ENABLED, False)

        # update status
        oldStatus = curation.get(STATUS)
        if STATUS in params:
            # verify valid status
            if params[STATUS] not in [CONSTRUCTION, REQUESTED, APPROVED]:
                raise RestException('Invalid status parameter')
            # regular users can only do construction -> requested transition
            if curation.get(STATUS) != CONSTRUCTION or \
               params[STATUS] != REQUESTED:
                self.requireAdmin(user)
            curation[STATUS] = params[STATUS]
        status = curation.get(STATUS)

        # admin enabling curation
        if enabled and not oldEnabled:
            # TODO: check if writers exist?
            # TODO: email writers?
            with self._progressContext(folder, TITLE_PRIVATE) as pc:
                self._setPublic(folder, False, pc)
            curation[ENABLE_USER_ID] = user.get('_id')
            self._addTimeline(oldCuration, curation, 'enabled curation')

        # admin reopening folder
        if enabled and oldStatus == APPROVED and status == CONSTRUCTION:
            with self._progressContext(folder, TITLE_PRIVATE) as pc:
                self._setPublic(folder, False, pc)
            curation[ENABLE_USER_ID] = user.get('_id')
            with self._progressContext(folder, TITLE_WRITEABLE) as pc:
                self._makeWriteable(folder, pc)
            self._addTimeline(oldCuration, curation, 'reopened folder')

        # admin disabling curation
        if not enabled and oldEnabled:
            self._addTimeline(oldCuration, curation, 'disabled curation')

        # user requesting approval
        if enabled and oldStatus == CONSTRUCTION and status == REQUESTED:
            curation[REQUEST_USER_ID] = user.get('_id')
            with self._progressContext(folder, TITLE_READ_ONLY) as pc:
                self._makeReadOnly(folder, pc)
            self._addTimeline(oldCuration, curation, 'requested approval')
            # send email to admin requesting approval
            self._sendMail(
                folder, curation.get(ENABLE_USER_ID),
                'REQUEST FOR APPROVAL: ' + folder['name'],
                'curation.requested.mako')

        # admin approving request
        if enabled and oldStatus == REQUESTED and status == APPROVED:
            with self._progressContext(folder, TITLE_PUBLIC) as pc:
                self._setPublic(folder, True, pc)
            curation[REVIEW_USER_ID] = user.get('_id')
            self._addTimeline(oldCuration, curation, 'approved request')
            # send approval notification to requestor
            self._sendMail(
                folder, curation.get(REQUEST_USER_ID),
                'APPROVED: ' + folder['name'],
                'curation.approved.mako')

        # admin rejecting request
        if enabled and oldStatus == REQUESTED and status == CONSTRUCTION:
            curation[REVIEW_USER_ID] = user.get('_id')
            with self._progressContext(folder, TITLE_WRITEABLE) as pc:
                self._makeWriteable(folder, pc)
            self._addTimeline(oldCuration, curation, 'rejected request')
            # send rejection notification to requestor
            self._sendMail(
                folder, curation.get(REQUEST_USER_ID),
                'REJECTED: ' + folder['name'],
                'curation.rejected.mako')

        folder = Folder().save(folder)
        curation['public'] = folder.get('public')
        return curation
Пример #42
0
 def validate(self, doc, **kwargs):
     # TODO: implement
     # raise ValidationException
     return Folder.validate(self, doc, **kwargs)
Пример #43
0
 def findOne(self, query=None, annotator_user=None, state=None, **kwargs):
     study_query = self._find_query_filter(query, annotator_user, state)
     return Folder.findOne(self, study_query, **kwargs)
Пример #44
0
class Folder(Resource):
    """API Endpoint for folders."""

    def __init__(self):
        super(Folder, self).__init__()
        self.resourceName = 'folder'
        self._model = FolderModel()
        self.route('DELETE', (':id',), self.deleteFolder)
        self.route('DELETE', (':id', 'contents'), self.deleteContents)
        self.route('GET', (), self.find)
        self.route('GET', (':id',), self.getFolder)
        self.route('GET', (':id', 'details'), self.getFolderDetails)
        self.route('GET', (':id', 'access'), self.getFolderAccess)
        self.route('GET', (':id', 'download'), self.downloadFolder)
        self.route('GET', (':id', 'rootpath'), self.rootpath)
        self.route('POST', (), self.createFolder)
        self.route('PUT', (':id',), self.updateFolder)
        self.route('PUT', (':id', 'access'), self.updateFolderAccess)
        self.route('POST', (':id', 'copy'), self.copyFolder)
        self.route('PUT', (':id', 'metadata'), self.setMetadata)
        self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)

    @access.public(scope=TokenScope.DATA_READ)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Search for folders by certain properties.')
        .notes('You must pass either a "folderId" or "text" field '
               'to specify how you are searching for folders.  '
               'If you omit one of these parameters the request will fail and respond : '
               '"Invalid search mode."')
        .responseClass('Folder', array=True)
        .param('parentType', "Type of the folder's parent", required=False,
               enum=['folder', 'user', 'collection'])
        .param('parentId', "The ID of the folder's parent.", required=False)
        .param('text', 'Pass to perform a text search.', required=False)
        .param('name', 'Pass to lookup a folder by exact name match. Must '
               'pass parentType and parentId as well when using this.', required=False)
        .pagingParams(defaultSort='lowerName')
        .errorResponse()
        .errorResponse('Read access was denied on the parent resource.', 403)
    )
    def find(self, parentType, parentId, text, name, limit, offset, sort):
        """
        Get a list of folders with given search parameters. Currently accepted
        search modes are:

        1. Searching by parentId and parentType, with optional additional
           filtering by the name field (exact match) or using full text search
           within a single parent folder. Pass a "name" parameter or "text"
           parameter to invoke these additional filters.
        2. Searching with full text search across all folders in the system.
           Simply pass a "text" parameter for this mode.
        """
        user = self.getCurrentUser()

        if parentType and parentId:
            parent = ModelImporter.model(parentType).load(
                parentId, user=user, level=AccessType.READ, exc=True)

            filters = {}
            if text:
                filters['$text'] = {
                    '$search': text
                }
            if name:
                filters['name'] = name

            return self._model.childFolders(
                parentType=parentType, parent=parent, user=user,
                offset=offset, limit=limit, sort=sort, filters=filters)
        elif text:
            return self._model.textSearch(
                text, user=user, limit=limit, offset=offset, sort=sort)
        else:
            raise RestException('Invalid search mode.')

    @access.public(scope=TokenScope.DATA_READ)
    @autoDescribeRoute(
        Description('Get detailed information about a folder.')
        .modelParam('id', model=FolderModel, level=AccessType.READ)
        .errorResponse()
        .errorResponse('Read access was denied on the folder.', 403)
    )
    def getFolderDetails(self, folder):
        return {
            'nItems': self._model.countItems(folder),
            'nFolders': self._model.countFolders(
                folder, user=self.getCurrentUser(), level=AccessType.READ)
        }

    @access.public(scope=TokenScope.DATA_READ, cookie=True)
    @autoDescribeRoute(
        Description('Download an entire folder as a zip archive.')
        .modelParam('id', model=FolderModel, level=AccessType.READ)
        .jsonParam('mimeFilter', 'JSON list of MIME types to include.', required=False,
                   requireArray=True)
        .produces('application/zip')
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the folder.', 403)
    )
    def downloadFolder(self, folder, mimeFilter):
        """
        Returns a generator function that will be used to stream out a zip
        file containing this folder's contents, filtered by permissions.
        """
        setResponseHeader('Content-Type', 'application/zip')
        setContentDisposition(folder['name'] + '.zip')
        user = self.getCurrentUser()

        def stream():
            zip = ziputil.ZipGenerator(folder['name'])
            for (path, file) in self._model.fileList(
                    folder, user=user, subpath=False, mimeFilter=mimeFilter):
                for data in zip.addFile(file, path):
                    yield data
            yield zip.footer()
        return stream

    @access.user(scope=TokenScope.DATA_WRITE)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Update a folder or move it into a new parent.')
        .responseClass('Folder')
        .modelParam('id', model=FolderModel, level=AccessType.WRITE)
        .param('name', 'Name of the folder.', required=False, strip=True)
        .param('description', 'Description for the folder.', required=False, strip=True)
        .param('parentType', "Type of the folder's parent", required=False,
               enum=['folder', 'user', 'collection'], strip=True)
        .param('parentId', 'Parent ID for the new parent of this folder.', required=False)
        .jsonParam('metadata', 'A JSON object containing the metadata keys to add',
                   paramType='form', requireObject=True, required=False)
        .errorResponse('ID was invalid.')
        .errorResponse('Write access was denied for the folder or its new parent object.', 403)
    )
    def updateFolder(self, folder, name, description, parentType, parentId, metadata):
        user = self.getCurrentUser()
        if name is not None:
            folder['name'] = name
        if description is not None:
            folder['description'] = description

        folder = self._model.updateFolder(folder)
        if metadata:
            folder = self._model.setMetadata(folder, metadata)

        if parentType and parentId:
            parent = ModelImporter.model(parentType).load(
                parentId, level=AccessType.WRITE, user=user, exc=True)
            if (parentType, parent['_id']) != (folder['parentCollection'], folder['parentId']):
                folder = self._model.move(folder, parent, parentType)

        return folder

    @access.user(scope=TokenScope.DATA_OWN)
    @filtermodel(model=FolderModel, addFields={'access'})
    @autoDescribeRoute(
        Description('Update the access control list for a folder.')
        .modelParam('id', model=FolderModel, level=AccessType.ADMIN)
        .jsonParam('access', 'The JSON-encoded access control list.', requireObject=True)
        .jsonParam('publicFlags', 'JSON list of public access flags.', requireArray=True,
                   required=False)
        .param('public', 'Whether the folder should be publicly visible.',
               dataType='boolean', required=False)
        .param('recurse', 'Whether the policies should be applied to all '
               'subfolders under this folder as well.', dataType='boolean',
               default=False, required=False)
        .param('progress', 'If recurse is set to True, this controls whether '
               'progress notifications will be sent.', dataType='boolean',
               default=False, required=False)
        .errorResponse('ID was invalid.')
        .errorResponse('Admin access was denied for the folder.', 403)
    )
    def updateFolderAccess(self, folder, access, publicFlags, public, recurse, progress):
        user = self.getCurrentUser()
        progress = progress and recurse  # Only enable progress in recursive case
        with ProgressContext(progress, user=user, title='Updating permissions',
                             message='Calculating progress...') as ctx:
            if progress:
                ctx.update(total=self._model.subtreeCount(
                    folder, includeItems=False, user=user, level=AccessType.ADMIN))
            return self._model.setAccessList(
                folder, access, save=True, recurse=recurse, user=user,
                progress=ctx, setPublic=public, publicFlags=publicFlags)

    @access.user(scope=TokenScope.DATA_WRITE)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Create a new folder.')
        .responseClass('Folder')
        .param('parentType', "Type of the folder's parent", required=False,
               enum=['folder', 'user', 'collection'], default='folder')
        .param('parentId', "The ID of the folder's parent.")
        .param('name', 'Name of the folder.', strip=True)
        .param('description', 'Description for the folder.', required=False,
               default='', strip=True)
        .param('reuseExisting', 'Return existing folder if it exists rather than '
               'creating a new one.', required=False,
               dataType='boolean', default=False)
        .param('public', 'Whether the folder should be publicly visible. By '
               'default, inherits the value from parent folder, or in the '
               'case of user or collection parentType, defaults to False.',
               required=False, dataType='boolean')
        .jsonParam('metadata', 'A JSON object containing the metadata keys to add',
                   paramType='form', requireObject=True, required=False)
        .errorResponse()
        .errorResponse('Write access was denied on the parent', 403)
    )
    def createFolder(self, public, parentType, parentId, name, description,
                     reuseExisting, metadata):
        user = self.getCurrentUser()
        parent = ModelImporter.model(parentType).load(
            id=parentId, user=user, level=AccessType.WRITE, exc=True)

        newFolder = self._model.createFolder(
            parent=parent, name=name, parentType=parentType, creator=user,
            description=description, public=public, reuseExisting=reuseExisting)
        if metadata:
            newFolder = self._model.setMetadata(newFolder, metadata)
        return newFolder

    @access.public(scope=TokenScope.DATA_READ)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Get a folder by ID.')
        .responseClass('Folder')
        .modelParam('id', model=FolderModel, level=AccessType.READ)
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the folder.', 403)
    )
    def getFolder(self, folder):
        return folder

    @access.user(scope=TokenScope.DATA_OWN)
    @autoDescribeRoute(
        Description('Get the access control list for a folder.')
        .responseClass('Folder')
        .modelParam('id', model=FolderModel, level=AccessType.ADMIN)
        .errorResponse('ID was invalid.')
        .errorResponse('Admin access was denied for the folder.', 403)
    )
    def getFolderAccess(self, folder):
        return self._model.getFullAccessList(folder)

    @access.user(scope=TokenScope.DATA_OWN)
    @autoDescribeRoute(
        Description('Delete a folder by ID.')
        .modelParam('id', model=FolderModel, level=AccessType.ADMIN)
        .param('progress', 'Whether to record progress on this task.',
               required=False, dataType='boolean', default=False)
        .errorResponse('ID was invalid.')
        .errorResponse('Admin access was denied for the folder.', 403)
    )
    def deleteFolder(self, folder, progress):
        with ProgressContext(progress, user=self.getCurrentUser(),
                             title='Deleting folder %s' % folder['name'],
                             message='Calculating folder size...') as ctx:
            # Don't do the subtree count if we weren't asked for progress
            if progress:
                ctx.update(total=self._model.subtreeCount(folder))
            self._model.remove(folder, progress=ctx)
        return {'message': 'Deleted folder %s.' % folder['name']}

    @access.user(scope=TokenScope.DATA_WRITE)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Set metadata fields on an folder.')
        .responseClass('Folder')
        .notes('Set metadata fields to null in order to delete them.')
        .modelParam('id', model=FolderModel, level=AccessType.WRITE)
        .jsonParam('metadata', 'A JSON object containing the metadata keys to add',
                   paramType='body', requireObject=True)
        .param('allowNull', 'Whether "null" is allowed as a metadata value.', required=False,
               dataType='boolean', default=False)
        .errorResponse(('ID was invalid.',
                        'Invalid JSON passed in request body.',
                        'Metadata key name was invalid.'))
        .errorResponse('Write access was denied for the folder.', 403)
    )
    def setMetadata(self, folder, metadata, allowNull):
        return self._model.setMetadata(folder, metadata, allowNull=allowNull)

    @access.user(scope=TokenScope.DATA_WRITE)
    @filtermodel(model=FolderModel)
    @autoDescribeRoute(
        Description('Copy a folder.')
        .responseClass('Folder')
        .modelParam('id', 'The ID of the original folder.', model=FolderModel,
                    level=AccessType.READ)
        .param('parentType', "Type of the new folder's parent", required=False,
               enum=['folder', 'user', 'collection'])
        .param('parentId', 'The ID of the parent document.', required=False)
        .param('name', 'Name for the new folder.', required=False)
        .param('description', "Description for the new folder.", required=False)
        .param('public', "Whether the folder should be publicly visible. By "
               "default, inherits the value from parent folder, or in the case "
               "of user or collection parentType, defaults to False. If "
               "'original', use the value of the original folder.",
               required=False, enum=['true', 'false', 'original'])
        .param('progress', 'Whether to record progress on this task.',
               required=False, dataType='boolean', default=False)
        .errorResponse(('A parameter was invalid.',
                        'ID was invalid.'))
        .errorResponse('Read access was denied on the original folder.\n\n'
                       'Write access was denied on the parent.', 403)
    )
    def copyFolder(self, folder, parentType, parentId, name, description, public, progress):
        user = self.getCurrentUser()
        parentType = parentType or folder['parentCollection']
        if parentId:
            parent = ModelImporter.model(parentType).load(
                id=parentId, user=user, level=AccessType.WRITE, exc=True)
        else:
            parent = None

        with ProgressContext(progress, user=self.getCurrentUser(),
                             title='Copying folder %s' % folder['name'],
                             message='Calculating folder size...') as ctx:
            # Don't do the subtree count if we weren't asked for progress
            if progress:
                ctx.update(total=self._model.subtreeCount(folder))
            return self._model.copyFolder(
                folder, creator=user, name=name, parentType=parentType,
                parent=parent, description=description, public=public, progress=ctx)

    @access.user(scope=TokenScope.DATA_WRITE)
    @autoDescribeRoute(
        Description('Remove all contents from a folder.')
        .notes('Cleans out all the items and subfolders from under a folder, '
               'but does not remove the folder itself.')
        .modelParam('id', 'The ID of the folder to clean.', model=FolderModel,
                    level=AccessType.WRITE)
        .param('progress', 'Whether to record progress on this task.',
               required=False, dataType='boolean', default=False)
        .errorResponse('ID was invalid.')
        .errorResponse('Write access was denied on the folder.', 403)
    )
    def deleteContents(self, folder, progress):
        with ProgressContext(progress, user=self.getCurrentUser(),
                             title='Clearing folder %s' % folder['name'],
                             message='Calculating folder size...') as ctx:
            # Don't do the subtree count if we weren't asked for progress
            if progress:
                ctx.update(total=self._model.subtreeCount(folder) - 1)
            self._model.clean(folder, progress=ctx)
        return {'message': 'Cleaned folder %s.' % folder['name']}

    @access.user(scope=TokenScope.DATA_WRITE)
    @filtermodel(FolderModel)
    @autoDescribeRoute(
        Description('Delete metadata fields on a folder.')
        .responseClass('Folder')
        .modelParam('id', model=FolderModel, level=AccessType.WRITE)
        .jsonParam(
            'fields', 'A JSON list containing the metadata fields to delete',
            paramType='body', schema={
                'type': 'array',
                'items': {
                    'type': 'string'
                }
            }
        )
        .errorResponse(('ID was invalid.',
                        'Invalid JSON passed in request body.',
                        'Metadata key name was invalid.'))
        .errorResponse('Write access was denied for the folder.', 403)
    )
    def deleteMetadata(self, folder, fields):
        return self._model.deleteMetadata(folder, fields)

    @access.public(scope=TokenScope.DATA_READ)
    @autoDescribeRoute(
        Description('Get the path to the root of the folder\'s hierarchy.')
        .modelParam('id', model=FolderModel, level=AccessType.READ)
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the folder.', 403)
    )
    def rootpath(self, folder, params):
        return self._model.parentsToRoot(folder, user=self.getCurrentUser())
Пример #45
0
 def findOne(self, query=None, **kwargs):
     dataset_query = self._find_query_filter(query)
     return Folder.findOne(self, dataset_query, **kwargs)
Пример #46
0
def _validateItem(event):
    parent = Folder().load(event.info['folderId'], force=True, exc=True)
    if parent.get('isVirtual'):
        raise ValidationException(
            'You may not place items under a virtual folder.', field='folderId')
Пример #47
0
    def testAccessFlags(self):
        resp = self.request('/system/access_flag')
        self.assertStatusOk(resp)
        self.assertEqual(resp.json, {})

        registerAccessFlag('my_key', name='hello', description='a custom flag')

        resp = self.request('/system/access_flag')
        self.assertStatusOk(resp)
        self.assertEqual(resp.json, {
            'my_key': {
                'name': 'hello',
                'description': 'a custom flag',
                'admin': False
            }
        })

        self.users[1] = User().load(self.users[1]['_id'], force=True)
        user = self.users[1]

        # Manage custom access flags on an access controlled resource
        self.assertFalse(User().hasAccessFlags(user, user, flags=['my_key']))

        # Admin should always have permission
        self.assertTrue(User().hasAccessFlags(user, self.users[0], flags=['my_key']))

        # Test the requireAccessFlags method
        with self.assertRaises(AccessException):
            User().requireAccessFlags(user, user=user, flags='my_key')

        User().requireAccessFlags(user, user=self.users[0], flags='my_key')

        acl = User().getFullAccessList(user)
        self.assertEqual(acl['users'][0]['flags'], [])

        # Test loadmodel requiredFlags argument via REST endpoint
        resp = self.request(
            '/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
        self.assertStatus(resp, 403)

        user = User().setAccessList(self.users[0], access={
            'users': [{
                'id': self.users[1]['_id'],
                'level': AccessType.ADMIN,
                'flags': ['my_key', 'not a registered flag']
            }],
            'groups': [{
                'id': self.group['_id'],
                'level': AccessType.ADMIN,
                'flags': ['my_key']
            }]
        }, save=True)

        resp = self.request(
            '/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
        self.assertStatusOk(resp)
        self.assertEqual(resp.json, 'success')

        # Only registered flags should be stored
        acl = User().getFullAccessList(user)
        self.assertEqual(acl['users'][0]['flags'], ['my_key'])
        self.assertTrue(User().hasAccessFlags(user, user, flags=['my_key']))

        # Create an admin-only access flag
        registerAccessFlag('admin_flag', name='admin flag', admin=True)

        # Non-admin shouldn't be able to set it
        user = User().setAccessList(self.users[0], access={
            'users': [{
                'id': self.users[1]['_id'],
                'level': AccessType.ADMIN,
                'flags': ['admin_flag']
            }],
            'groups': []
        }, save=True, user=self.users[1])

        acl = User().getFullAccessList(user)
        self.assertEqual(acl['users'][0]['flags'], [])

        # Admin user should be able to set it
        user = User().setAccessList(self.users[1], access={
            'users': [{
                'id': self.users[1]['_id'],
                'level': AccessType.ADMIN,
                'flags': ['admin_flag']
            }],
            'groups': [{
                'id': self.group['_id'],
                'level': AccessType.ADMIN,
                'flags': ['admin_flag']
            }]
        }, save=True, user=self.users[0])

        acl = User().getFullAccessList(user)
        self.assertEqual(acl['users'][0]['flags'], ['admin_flag'])

        # An already-enabled admin-only flag should stay enabled for non-admin user
        user = User().setAccessList(self.users[1], access={
            'users': [{
                'id': self.users[1]['_id'],
                'level': AccessType.ADMIN,
                'flags': ['my_key', 'admin_flag']
            }],
            'groups': [{
                'id': self.group['_id'],
                'level': AccessType.ADMIN,
                'flags': ['admin_flag']
            }]
        }, save=True, user=self.users[1])

        acl = User().getFullAccessList(user)
        self.assertEqual(set(acl['users'][0]['flags']), {'my_key', 'admin_flag'})
        self.assertEqual(acl['groups'][0]['flags'], ['admin_flag'])

        # Test setting public flags on a collection and folder
        collectionModel = Collection()
        folderModel = Folder()
        itemModel = Item()
        collection = collectionModel.createCollection('coll', creator=self.users[0], public=True)
        folder = folderModel.createFolder(
            collection, 'folder', parentType='collection', creator=self.users[0])

        # Add an item to the folder so we can test AclMixin flag behavior
        item = itemModel.createItem(folder=folder, name='test', creator=self.users[0])

        folder = folderModel.setUserAccess(
            folder, self.users[1], level=AccessType.ADMIN, save=True, currentUser=self.users[0])

        with self.assertRaises(AccessException):
            collectionModel.requireAccessFlags(collection, user=None, flags='my_key')

        # Test AclMixin flag behavior
        with self.assertRaises(AccessException):
            itemModel.requireAccessFlags(item, user=None, flags='my_key')

        self.assertFalse(itemModel.hasAccessFlags(item, user=None, flags='my_key'))

        collection = collectionModel.setAccessList(
            collection, access=collection['access'], save=True, recurse=True, user=self.users[0],
            publicFlags=['my_key'])
        collectionModel.requireAccessFlags(collection, user=None, flags='my_key')

        # Make sure recursive setting of public flags worked
        folder = folderModel.load(folder['_id'], force=True)
        self.assertEqual(folder['publicFlags'], ['my_key'])

        itemModel.requireAccessFlags(item, user=None, flags='my_key')

        # Non-admin shouldn't be able to set admin-only public flags
        folder = folderModel.setPublicFlags(
            folder, flags=['admin_flag'], user=self.users[1], save=True)
        self.assertEqual(folder['publicFlags'], [])

        # Admin users should be able to set admin-only public flags
        folder = folderModel.setPublicFlags(
            folder, flags=['admin_flag'], user=self.users[0], save=True, append=True)
        self.assertEqual(folder['publicFlags'], ['admin_flag'])

        # Non-admin users can set admin-only public flags if they are already enabled
        folder = folderModel.setPublicFlags(
            folder, flags=['admin_flag', 'my_key'], user=self.users[1], save=True)
        self.assertEqual(set(folder['publicFlags']), {'admin_flag', 'my_key'})

        # Test "force" options
        folder = folderModel.setPublicFlags(folder, flags='admin_flag', force=True, save=True)
        self.assertEqual(folder['publicFlags'], ['admin_flag'])

        folder = folderModel.setAccessList(folder, access={
            'users': [{
                'id': self.users[1]['_id'],
                'level': AccessType.ADMIN,
                'flags': ['my_key', 'admin_flag']
            }],
            'groups': []
        }, save=True, force=True)
        folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')

        folder = folderModel.setUserAccess(
            folder, self.users[1], level=AccessType.READ, save=True, force=True, flags=[])
        self.assertFalse(folderModel.hasAccessFlags(folder, self.users[1], flags='my_key'))

        folder = folderModel.setGroupAccess(
            folder, self.group, level=AccessType.READ, save=True, force=True, flags='my_key')
        folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')

        # Testing with flags=None should give sensible behavior
        folderModel.requireAccessFlags(folder, user=None, flags=None)

        # Test filtering results by access flags (both ACModel and AclMixin)
        for model, doc in ((folderModel, folder), (itemModel, item)):
            cursor = model.find({})
            self.assertGreater(len(list(cursor)), 0)

            cursor = model.find({})
            filtered = list(model.filterResultsByPermission(
                cursor, user=None, level=AccessType.READ, flags='my_key'))
            self.assertEqual(len(filtered), 0)

            cursor = model.find({})
            filtered = list(model.filterResultsByPermission(
                cursor, user=self.users[1], level=AccessType.READ, flags=('my_key', 'admin_flag')))
            self.assertEqual(len(filtered), 1)
            self.assertEqual(filtered[0]['_id'], doc['_id'])
Пример #48
0
    def testFolderAccessAndDetails(self):
        # create a folder to work with
        folder = Folder().createFolder(
            parent=self.admin, parentType='user', creator=self.admin,
            name='Folder')

        resp = self.request(
            path='/folder/%s/access' % folder['_id'], method='GET',
            user=self.admin)
        self.assertStatusOk(resp)
        access = resp.json
        self.assertEqual(access, {
            'users': [{
                'login': self.admin['login'],
                'level': AccessType.ADMIN,
                'id': str(self.admin['_id']),
                'flags': [],
                'name': '%s %s' % (
                    self.admin['firstName'], self.admin['lastName'])}],
            'groups': []
        })
        self.assertTrue(not folder.get('public'))
        # Setting the access list with bad json should throw an error
        resp = self.request(
            path='/folder/%s/access' % folder['_id'], method='PUT',
            user=self.admin, params={'access': 'badJSON'})
        self.assertStatus(resp, 400)
        # Change the access to public
        resp = self.request(
            path='/folder/%s/access' % folder['_id'], method='PUT',
            user=self.admin,
            params={'access': json.dumps(access), 'public': True})
        self.assertStatusOk(resp)
        resp = self.request(
            path='/folder/%s' % folder['_id'], method='GET',
            user=self.admin)
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['public'], True)

        # Create an item in the folder
        Item().createItem(folder=folder, creator=self.admin, name='Item')
        # Create a public and private folder within the folder
        Folder().createFolder(
            parent=folder, parentType='folder', creator=self.admin,
            name='Public', public=True)
        Folder().createFolder(
            parent=folder, parentType='folder', creator=self.admin,
            name='Private', public=False)

        # Test folder details as anonymous
        resp = self.request(
            path='/folder/%s/details' % str(folder['_id']))
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['nItems'], 1)
        self.assertEqual(resp.json['nFolders'], 1)

        # Test folder details as admin
        resp = self.request(
            path='/folder/%s/details' % str(folder['_id']), user=self.admin)
        self.assertStatusOk(resp)
        self.assertEqual(resp.json['nItems'], 1)
        self.assertEqual(resp.json['nFolders'], 2)
Пример #49
0
class HdfsAssetstoreResource(Resource):
    def __init__(self):
        super(HdfsAssetstoreResource, self).__init__()
        self.resourceName = 'hdfs_assetstore'
        self.route('PUT', (':id', 'import'), self.importData)

        self.folderModel = Folder()  # Save to avoid many lookups
        self.itemModel = Item()
        self.fileModel = File()

    def _importFile(self, parent, name, user, assetstore, node):
        item = self.itemModel.findOne({
            'folderId': parent['_id'],
            'name': name
        })
        if item is None:
            item = self.itemModel.createItem(
                name=name, creator=user, folder=parent)

        file = self.fileModel.findOne({
            'name': name,
            'itemId': item['_id']
        })
        if file is None:
            file = self.fileModel.createFile(
                creator=user, item=item, name=name, size=node['length'],
                assetstore=assetstore, mimeType=None, saveFile=False)

        file['hdfs'] = {
            'imported': True,
            'path': node['path']
        }
        self.fileModel.save(file)

    def _importData(self, parentType, parent, assetstore, client, path, ctx,
                    user):
        for node in client.ls([path]):
            ctx.update(message='Importing ' + node['path'])
            name = posixpath.basename(node['path'])

            if node['file_type'] == 'd':
                folder = self.folderModel.findOne({
                    'parentId': parent['_id'],
                    'name': name,
                    'parentCollection': parentType
                })
                if folder is None:
                    folder = self.folderModel.createFolder(
                        parent, name, parentType=parentType, creator=user)

                self._importData('folder', folder, assetstore, client,
                                 node['path'], ctx, user)
            elif node['file_type'] == 'f' and parentType == 'folder':
                self._importFile(parent, name, user, assetstore, node)

    @access.admin
    @loadmodel(model='assetstore')
    @describeRoute(
        Description('Import a data hierarchy from an HDFS instance.')
        .notes('Only site administrators may use this endpoint.')
        .param('id', 'The ID of the assetstore representing the HDFS instance.',
               paramType='path')
        .param('parentId', 'The ID of the parent object in the Girder data '
               'hierarchy under which to import the files.')
        .param('parentType', 'The type of the parent object to import into.',
               enum=('folder', 'user', 'collection'), required=False)
        .param('path', 'Root of the directory structure (relative to the root '
               'of the HDFS) to import.')
        .param('progress', 'Whether to record progress on this operation ('
               'default=False)', required=False, dataType='boolean')
        .errorResponse()
        .errorResponse('You are not an administrator.', 403)
    )
    def importData(self, assetstore, params):
        self.requireParams(('parentId', 'path'), params)

        user = self.getCurrentUser()
        parentType = params.get('parentType', 'folder')
        if parentType not in ('user', 'collection', 'folder'):
            raise RestException('Invalid parentType.')

        parent = self.model(parentType).load(params['parentId'], force=True, exc=True)

        progress = self.boolParam('progress', params, default=False)
        client = HdfsClient(
            host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False)
        path = params['path']

        with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx:
            try:
                self._importData(parentType, parent, assetstore, client, path, ctx, user)
            except FileNotFoundException:
                raise RestException('File not found: %s.' % path)
Пример #50
0

class GeoParser(object):

    @cherrypy.expose
    def index(self):
        tmpl = lookup.get_template("index.html")
        return tmpl.render(salutation="Hello", target="World")



if __name__ == '__main__':

    girderUser = User()
    girderCollection = Collection()
    girderFolder = Folder()

    try:
        currentUser = girderUser.getAdmins()[0]
    except:
        print "ERROR: No user registered or logged in."
    existinig_collections = [each['name'] for each in girderCollection.list()]

    if len(existinig_collections) == 0 or not GIRDER_COLLECTION in existinig_collections:
        girderCollection.createCollection(GIRDER_COLLECTION, currentUser)
        existinig_collections = [each['name'] for each in girderCollection.list()]
    else:
        print "ERROR: Cannot create Collection. User not registered or logged in."

    try:
        for collection in girderCollection.list():