def _filter_items(folders, user, query, matches, parent): for folder in folders: items = list(Folder().childItems(folder, parentType='folder', user=user)) for item in items: full_path = getResourcePath('item', item, user) trimmed_path = full_path.split(parent['name'])[1] if query in trimmed_path: matches.append({'text': trimmed_path, 'value': item}) folders = list(Folder().childFolders(folder, parentType='folder', user=user)) if folders: _filter_items(folders, user, query, matches, parent) return matches
def getServerFusePath(name, type, doc): """ Given a fuse name and a resource, return the file path. :param name: key used for the fuse mount. :param type: the resource model type. :param doc: the resource document. :return: a path to the resource. """ if name not in _fuseMounts: return None return _fuseMounts[name]['path'].rstrip('/') + path_util.getResourcePath( type, doc, user=_fuseMounts[name]['user'], force=_fuseMounts[name]['force'])
def getServerFusePath(name, type, doc): """ Given a fuse name and a resource, return the file path. :param name: key used for the fuse mount. :param type: the resource model type. :param doc: the resource document. :return: a path to the resource. """ if name not in _fuseMounts: return None return _fuseMounts[name]['path'].rstrip('/') + path_util.getResourcePath( type, doc, user=_fuseMounts[name]['user'], force=_fuseMounts[name]['force'])
def path(self, event, path, root, user=None): res_type = event.info["params"]["type"] try: if res_type == "folder": self.is_dir(path, root) elif res_type in ("item", "file"): self.is_file(path, root) else: raise ValidationException("Not a file, nor a folder") except ValidationException: raise RestException("Invalid resource id.") root_girder_path = pathlib.Path(getResourcePath("folder", root, user=user)) remainder_path = path.relative_to(pathlib.PosixPath(root["fsPath"])) response = (root_girder_path / remainder_path).as_posix() event.preventDefault().addResponse(response)
def getGirderMountFilePath(self, file, validate=True): """ If possible, get the path of the file on a local girder mount. :param file: The file document. :param validate: if True, check if the path exists and raise an exception if it does not. :returns: a girder mount path to the file or None if no such path is available. """ mount = Setting().get(SettingKey.GIRDER_MOUNT_INFORMATION, None) if mount: path = mount['path'].rstrip('/') + path_util.getResourcePath('file', file, force=True) if not validate or os.path.exists(path): return path if validate: raise FilePathException('This file isn\'t accessible from a Girder mount.')
def getGirderMountFilePath(self, file, validate=True): """ If possible, get the path of the file on a local girder mount. :param file: The file document. :param validate: if True, check if the path exists and raise an exception if it does not. :returns: a girder mount path to the file or None if no such path is available. """ mount = Setting().get(SettingKey.GIRDER_MOUNT_INFORMATION, None) if mount: path = mount['path'].rstrip('/') + path_util.getResourcePath('file', file, force=True) if not validate or os.path.exists(path): return path if validate: raise FilePathException('This file isn\'t accessible from a Girder mount.')
def downloadFiles(assetstoreId): ids = [] print("Downloading all files from assetstore %s" % assetstoreId) files = getFiles(assetstoreId) i = 0 for file in files: for user in User().find({'_id': file["creatorId"]}): if user['login'] == 'admin': continue if file['itemId'] is not None: item = Item().load(file['itemId'], force=True) if item['baseParentType'] != "user": continue if 'copied' in file: continue print("Downloading files for user %s" % (user)) print("File %s" % file) fullpath = getResourcePath("file", file, user=user) fullpath = os.path.dirname(fullpath) if not fullpath.startswith("/user"): continue path = os.path.dirname(fullpath) os.makedirs(path, exist_ok=True) print("Downloading file %s" % (fullpath)) for data in File().download(file, headers=False)(): f = open(fullpath, "wb") f.write(data) f.close() i = i + 1 ids.append(file["_id"]) else: print("Item id is None for %s" % file['_id']) print("Downloaded %d of %d files" % (i, len(files))) return ids
def updateWorkspaceModTime(self, event): """ Handler for model.file.save, model.file.save.created and model.file.remove events When files in a workspace are modified or deleted, update the associated Tale with a workspaceModified time. This is used to determine whether to rebuild or not. """ # Get the path path = getResourcePath('file', event.info, force=True) # If the file is in a workspace, parse the Tale ID # e.g., "/collection/WholeTale Workspaces/ # WholeTale Workspaces/5c848784912a470001e9545d/file.txt" if path.startswith( '/collection/WholeTale Workspaces/WholeTale Workspaces'): elems = path.split('/') taleId = elems[4] tale = self.model('tale', 'wholetale').load(taleId, force=True) tale['workspaceModified'] = int(time.time()) self.model('tale', 'wholetale').save(tale)
def path(self, id, type): user = self.getCurrentUser() doc = self._getResource(id, type) if doc is None: raise RestException('Invalid resource id.') return path_util.getResourcePath(type, doc, user=user)
def testTaleFlow(self): resp = self.request( path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps({'imageId': str(self.image['_id'])}) ) self.assertStatus(resp, 400) self.assertEqual(resp.json, { 'message': ("Invalid JSON object for parameter tale: " "'involatileData' " "is a required property"), 'type': 'rest' }) # Grab the default user folders resp = self.request( path='/folder', method='GET', user=self.user, params={ 'parentType': 'user', 'parentId': self.user['_id'], 'sort': 'title', 'sortdir': 1 }) privateFolder = resp.json[0] publicFolder = resp.json[1] resp = self.request( path='/folder', method='GET', user=self.admin, params={ 'parentType': 'user', 'parentId': self.admin['_id'], 'sort': 'title', 'sortdir': 1 }) # adminPrivateFolder = resp.json[0] adminPublicFolder = resp.json[1] resp = self.request( path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'involatileData': [ {'type': 'folder', 'id': publicFolder['_id']} ] }) ) self.assertStatusOk(resp) tale = resp.json # Check that workspace was created # Check that data folder was created from girder.plugins.wholetale.constants import DATADIRS_NAME from girder.utility.path import getResourcePath from girder.models.folder import Folder sc = { '_id': tale['_id'], 'cname': DATADIRS_NAME, 'fname': DATADIRS_NAME } self.assertEqual( getResourcePath( 'folder', Folder().load(tale['folderId'], user=self.user), user=self.admin), '/collection/{cname}/{fname}/{_id}'.format(**sc) ) resp = self.request( path='/tale/{_id}'.format(**tale), method='PUT', type='application/json', user=self.user, body=json.dumps({ 'folderId': tale['folderId'], 'involatileData': tale['involatileData'], 'imageId': tale['imageId'], 'title': 'new name', 'description': 'new description', 'config': {'memLimit': '2g'}, 'public': True, 'published': False }) ) self.assertStatusOk(resp) self.assertEqual(resp.json['title'], 'new name') tale = resp.json resp = self.request( path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'involatileData': [ {'type': 'folder', 'id': privateFolder['_id']} ] }) ) self.assertStatusOk(resp) new_tale = resp.json resp = self.request( path='/tale', method='POST', user=self.admin, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'involatileData': [ {'type': 'folder', 'id': adminPublicFolder['_id']} ], 'public': False }) ) self.assertStatusOk(resp) # admin_tale = resp.json resp = self.request( path='/tale', method='GET', user=self.admin, params={} ) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 3) resp = self.request( path='/tale', method='GET', user=self.user, params={'imageId': str(self.image['_id'])} ) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 2) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id'], new_tale['_id']}) resp = self.request( path='/tale', method='GET', user=self.user, params={'userId': str(self.user['_id'])} ) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 2) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id'], new_tale['_id']}) resp = self.request( path='/tale', method='GET', user=self.user, params={'text': 'new'} ) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id']}) resp = self.request( path='/tale/{_id}'.format(**new_tale), method='DELETE', user=self.admin) self.assertStatusOk(resp) resp = self.request( path='/tale/{_id}'.format(**new_tale), method='GET', user=self.user) self.assertStatus(resp, 400) resp = self.request( path='/tale/{_id}'.format(**tale), method='GET', user=self.user) self.assertStatusOk(resp) for key in tale.keys(): if key in ('access', 'updated', 'created'): continue self.assertEqual(resp.json[key], tale[key]) resp = self.request( path='/tale/{_id}/export'.format(**tale), method='GET', user=self.user, type='application/octet-stream', isJson=False) self.assertStatus(resp, 200)
def path(self, id, type): user = self.getCurrentUser() doc = self._getResource(id, type) if doc is None: raise RestException('Invalid resource id.') return path_util.getResourcePath(type, doc, user=user)
def testTaleFlow(self): from server.lib.license import WholeTaleLicense resp = self.request(path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps( {'imageId': str(self.image['_id'])})) self.assertStatus(resp, 400) self.assertEqual( resp.json, { 'message': ("Invalid JSON object for parameter tale: " "'dataSet' " "is a required property"), 'type': 'rest' }) resp = self.request(path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'dataSet': [] })) self.assertStatusOk(resp) tale = resp.json # Check that workspace was created # Check that data folder was created from girder.plugins.wholetale.constants import DATADIRS_NAME from girder.utility.path import getResourcePath sc = { '_id': tale['_id'], 'cname': DATADIRS_NAME, 'fname': DATADIRS_NAME } self.assertEqual( getResourcePath('folder', Folder().load(tale['folderId'], user=self.user), user=self.admin), '/collection/{cname}/{fname}/{_id}'.format(**sc)) taleLicense = WholeTaleLicense.default_spdx() resp = self.request(path='/tale/{_id}'.format(**tale), method='PUT', type='application/json', user=self.user, body=json.dumps({ 'folderId': tale['folderId'], 'dataSet': tale['dataSet'], 'imageId': tale['imageId'], 'title': 'new name', 'description': 'new description', 'config': { 'memLimit': '2g' }, 'public': False, 'licenseSPDX': taleLicense, 'publishInfo': [{ 'pid': 'published_pid', 'uri': 'published_url', 'date': '2019-01-23T15:48:17.476000+00:00', }] })) self.assertStatusOk(resp) self.assertEqual(resp.json['title'], 'new name') self.assertEqual(resp.json['licenseSPDX'], taleLicense) tale = resp.json resp = self.request(path='/tale', method='POST', user=self.user, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'dataSet': [], })) self.assertStatusOk(resp) new_tale = resp.json resp = self.request(path='/tale', method='POST', user=self.admin, type='application/json', body=json.dumps({ 'imageId': str(self.image['_id']), 'dataSet': [], 'public': False })) self.assertStatusOk(resp) # admin_tale = resp.json resp = self.request(path='/tale', method='GET', user=self.admin, params={}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 3) resp = self.request(path='/tale', method='GET', user=self.user, params={'imageId': str(self.image['_id'])}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 2) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id'], new_tale['_id']}) resp = self.request(path='/tale', method='GET', user=self.user, params={'userId': str(self.user['_id'])}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 2) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id'], new_tale['_id']}) resp = self.request(path='/tale', method='GET', user=self.user, params={'text': 'new'}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id']}) resp = self.request(path='/tale/{_id}'.format(**new_tale), method='DELETE', user=self.admin) self.assertStatusOk(resp) resp = self.request(path='/tale/{_id}'.format(**new_tale), method='GET', user=self.user) self.assertStatus(resp, 400) resp = self.request(path='/tale/{_id}'.format(**tale), method='GET', user=self.user) self.assertStatusOk(resp) for key in tale.keys(): if key in ('access', 'updated', 'created'): continue self.assertEqual(resp.json[key], tale[key])
def getPathFromRoot(self, user, itemId): item = self.itemModel.load(itemId, user=user, level=AccessType.READ) return path_util.getResourcePath('item', item, user=user)
def sync(): import os from bson.objectid import ObjectId from girder import logger from girder.models.assetstore import Assetstore from girder.models.collection import Collection from girder.models.file import File from girder.models.folder import Folder from girder.models.item import Item from girder.models.user import User from girder.exceptions import ResourcePathNotFound from girder.utility import path as path_utils from girder.utility import assetstore_utilities assetstoreId = ObjectId('59b04a2c38eed90001dcc45c') assetstore = Assetstore().load(id=assetstoreId) adapter = assetstore_utilities.getAssetstoreAdapter(assetstore) physicalPath = '/mnt/data/renaissance' collectionPath = '/collection/Renaissance Simulations' collectionId = ObjectId('59b04a0e38eed90001dcc45b') rslCollection = Collection().load(collectionId, force=True) admin = list(User().getAdmins())[0] def purge_leaf_folder(path): folder = path_utils.lookUpPath(path, user=admin)['document'] if Item().find({'folderId': folder['_id']}).count() > 0 or \ list(Folder().childFolders(folder, 'folder', user=admin)): return logger.info("Removing empty folder %s" % path) Folder().remove(folder) purge_leaf_folder(os.path.dirname(path)) q = {'assetstoreId': assetstoreId, 'imported': True} fields = ['path', 'size', 'name', 'mtime'] girderFiles = { fObj.pop('path'): fObj for fObj in File().find(q, fields=fields) } toImport = {} toModify = {} for (dirpath, dirnames, filenames) in os.walk(physicalPath): for filename in filenames: path = os.path.join(dirpath, filename) stat = os.stat(path) try: girderObj = girderFiles.pop(path) except KeyError: toImport[path] = { 'mtime': stat.st_mtime, 'size': stat.st_size, 'name': filename } if girderObj['mtime'] != stat.st_mtime or \ girderObj['size'] != stat.st_size or \ girderObj['name'] != filename: girderObj.update({ 'mtime': stat.st_mtime, 'size': stat.st_size, 'name': filename, 'path': path }) toModify[girderObj.pop('_id')] = girderObj # Remove orphaned files potentialLeafFolders = set() for orphan in girderFiles.values(): fileObj = File().load(ObjectId(orphan['_id']), force=True) itemObj = Item().load(fileObj['itemId'], force=True) File().remove(fileObj) girderPath = path_utils.getResourcePath('item', itemObj, force=True) if not list(Item().childFiles(itemObj)): Item().remove(itemObj) potentialLeafFolders.add(os.path.dirname(girderPath)) logger.info('Removed %s' % girderPath) # Remove empty folders for path in list(potentialLeafFolders): purge_leaf_folder(path) # Import new items for filePath, newFile in toImport.items(): relpath = os.path.relpath(filePath, physicalPath) parentType = 'collection' parent = rslCollection dirs = os.path.dirname(relpath).split('/') for directory in dirs: try: parent, parentType = \ path_utils.lookUpToken(directory, parentType, parent) except ResourcePathNotFound: parent = Folder().createFolder(parent, directory, parentType=parentType, public=True, creator=admin) parentType = 'folder' adapter._importDataAsItem(os.path.basename(relpath), admin, parent, os.path.dirname(filePath), [os.path.basename(filePath)], reuseExisting=True) logger.info('Imported %s to %s' % (filePath, os.path.join(collectionPath, relpath)))