def _filter_items(folders, user, query, matches, parent):
for folder in folders:
items = list(Folder().childItems(folder, parentType='folder', user=user))
for item in items:
full_path = getResourcePath('item', item, user)
trimmed_path = full_path.split(parent['name'])[1]
if query in trimmed_path:
matches.append({'text': trimmed_path, 'value': item})
folders = list(Folder().childFolders(folder, parentType='folder', user=user))
if folders:
_filter_items(folders, user, query, matches, parent)
return matches
def getServerFusePath(name, type, doc):
"""
Given a fuse name and a resource, return the file path.
:param name: key used for the fuse mount.
:param type: the resource model type.
:param doc: the resource document.
:return: a path to the resource.
"""
if name not in _fuseMounts:
return None
return _fuseMounts[name]['path'].rstrip('/') + path_util.getResourcePath(
type, doc, user=_fuseMounts[name]['user'], force=_fuseMounts[name]['force'])
def getServerFusePath(name, type, doc):
"""
Given a fuse name and a resource, return the file path.
:param name: key used for the fuse mount.
:param type: the resource model type.
:param doc: the resource document.
:return: a path to the resource.
"""
if name not in _fuseMounts:
return None
return _fuseMounts[name]['path'].rstrip('/') + path_util.getResourcePath(
type,
doc,
user=_fuseMounts[name]['user'],
force=_fuseMounts[name]['force'])
def path(self, event, path, root, user=None):
res_type = event.info["params"]["type"]
try:
if res_type == "folder":
self.is_dir(path, root)
elif res_type in ("item", "file"):
self.is_file(path, root)
else:
raise ValidationException("Not a file, nor a folder")
except ValidationException:
raise RestException("Invalid resource id.")
root_girder_path = pathlib.Path(getResourcePath("folder", root, user=user))
remainder_path = path.relative_to(pathlib.PosixPath(root["fsPath"]))
response = (root_girder_path / remainder_path).as_posix()
event.preventDefault().addResponse(response)
def getGirderMountFilePath(self, file, validate=True):
"""
If possible, get the path of the file on a local girder mount.
:param file: The file document.
:param validate: if True, check if the path exists and raise an
exception if it does not.
:returns: a girder mount path to the file or None if no such path is
available.
"""
mount = Setting().get(SettingKey.GIRDER_MOUNT_INFORMATION, None)
if mount:
path = mount['path'].rstrip('/') + path_util.getResourcePath('file', file, force=True)
if not validate or os.path.exists(path):
return path
if validate:
raise FilePathException('This file isn\'t accessible from a Girder mount.')
def getGirderMountFilePath(self, file, validate=True):
"""
If possible, get the path of the file on a local girder mount.
:param file: The file document.
:param validate: if True, check if the path exists and raise an
exception if it does not.
:returns: a girder mount path to the file or None if no such path is
available.
"""
mount = Setting().get(SettingKey.GIRDER_MOUNT_INFORMATION, None)
if mount:
path = mount['path'].rstrip('/') + path_util.getResourcePath('file', file, force=True)
if not validate or os.path.exists(path):
return path
if validate:
raise FilePathException('This file isn\'t accessible from a Girder mount.')
def downloadFiles(assetstoreId):
ids = []
print("Downloading all files from assetstore %s" % assetstoreId)
files = getFiles(assetstoreId)
i = 0
for file in files:
for user in User().find({'_id': file["creatorId"]}):
if user['login'] == 'admin':
continue
if file['itemId'] is not None:
item = Item().load(file['itemId'], force=True)
if item['baseParentType'] != "user":
continue
if 'copied' in file:
continue
print("Downloading files for user %s" % (user))
print("File %s" % file)
fullpath = getResourcePath("file", file, user=user)
fullpath = os.path.dirname(fullpath)
if not fullpath.startswith("/user"):
continue
path = os.path.dirname(fullpath)
os.makedirs(path, exist_ok=True)
print("Downloading file %s" % (fullpath))
for data in File().download(file, headers=False)():
f = open(fullpath, "wb")
f.write(data)
f.close()
i = i + 1
ids.append(file["_id"])
else:
print("Item id is None for %s" % file['_id'])
print("Downloaded %d of %d files" % (i, len(files)))
return ids
def updateWorkspaceModTime(self, event):
"""
Handler for model.file.save, model.file.save.created and
model.file.remove events When files in a workspace are modified or
deleted, update the associated Tale with a workspaceModified time.
This is used to determine whether to rebuild or not.
"""
# Get the path
path = getResourcePath('file', event.info, force=True)
# If the file is in a workspace, parse the Tale ID
# e.g., "/collection/WholeTale Workspaces/
# WholeTale Workspaces/5c848784912a470001e9545d/file.txt"
if path.startswith(
'/collection/WholeTale Workspaces/WholeTale Workspaces'):
elems = path.split('/')
taleId = elems[4]
tale = self.model('tale', 'wholetale').load(taleId, force=True)
tale['workspaceModified'] = int(time.time())
self.model('tale', 'wholetale').save(tale)
def path(self, id, type):
user = self.getCurrentUser()
doc = self._getResource(id, type)
if doc is None:
raise RestException('Invalid resource id.')
return path_util.getResourcePath(type, doc, user=user)
def testTaleFlow(self):
resp = self.request(
path='/tale', method='POST', user=self.user,
type='application/json',
body=json.dumps({'imageId': str(self.image['_id'])})
)
self.assertStatus(resp, 400)
self.assertEqual(resp.json, {
'message': ("Invalid JSON object for parameter tale: "
"'involatileData' "
"is a required property"),
'type': 'rest'
})
# Grab the default user folders
resp = self.request(
path='/folder', method='GET', user=self.user, params={
'parentType': 'user',
'parentId': self.user['_id'],
'sort': 'title',
'sortdir': 1
})
privateFolder = resp.json[0]
publicFolder = resp.json[1]
resp = self.request(
path='/folder', method='GET', user=self.admin, params={
'parentType': 'user',
'parentId': self.admin['_id'],
'sort': 'title',
'sortdir': 1
})
# adminPrivateFolder = resp.json[0]
adminPublicFolder = resp.json[1]
resp = self.request(
path='/tale', method='POST', user=self.user,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'involatileData': [
{'type': 'folder', 'id': publicFolder['_id']}
]
})
)
self.assertStatusOk(resp)
tale = resp.json
# Check that workspace was created
# Check that data folder was created
from girder.plugins.wholetale.constants import DATADIRS_NAME
from girder.utility.path import getResourcePath
from girder.models.folder import Folder
sc = {
'_id': tale['_id'],
'cname': DATADIRS_NAME,
'fname': DATADIRS_NAME
}
self.assertEqual(
getResourcePath(
'folder',
Folder().load(tale['folderId'], user=self.user),
user=self.admin),
'/collection/{cname}/{fname}/{_id}'.format(**sc)
)
resp = self.request(
path='/tale/{_id}'.format(**tale), method='PUT',
type='application/json',
user=self.user, body=json.dumps({
'folderId': tale['folderId'],
'involatileData': tale['involatileData'],
'imageId': tale['imageId'],
'title': 'new name',
'description': 'new description',
'config': {'memLimit': '2g'},
'public': True,
'published': False
})
)
self.assertStatusOk(resp)
self.assertEqual(resp.json['title'], 'new name')
tale = resp.json
resp = self.request(
path='/tale', method='POST', user=self.user,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'involatileData': [
{'type': 'folder', 'id': privateFolder['_id']}
]
})
)
self.assertStatusOk(resp)
new_tale = resp.json
resp = self.request(
path='/tale', method='POST', user=self.admin,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'involatileData': [
{'type': 'folder', 'id': adminPublicFolder['_id']}
],
'public': False
})
)
self.assertStatusOk(resp)
# admin_tale = resp.json
resp = self.request(
path='/tale', method='GET', user=self.admin,
params={}
)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 3)
resp = self.request(
path='/tale', method='GET', user=self.user,
params={'imageId': str(self.image['_id'])}
)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 2)
self.assertEqual(set([_['_id'] for _ in resp.json]),
{tale['_id'], new_tale['_id']})
resp = self.request(
path='/tale', method='GET', user=self.user,
params={'userId': str(self.user['_id'])}
)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 2)
self.assertEqual(set([_['_id'] for _ in resp.json]),
{tale['_id'], new_tale['_id']})
resp = self.request(
path='/tale', method='GET', user=self.user,
params={'text': 'new'}
)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(set([_['_id'] for _ in resp.json]),
{tale['_id']})
resp = self.request(
path='/tale/{_id}'.format(**new_tale), method='DELETE',
user=self.admin)
self.assertStatusOk(resp)
resp = self.request(
path='/tale/{_id}'.format(**new_tale), method='GET',
user=self.user)
self.assertStatus(resp, 400)
resp = self.request(
path='/tale/{_id}'.format(**tale), method='GET',
user=self.user)
self.assertStatusOk(resp)
for key in tale.keys():
if key in ('access', 'updated', 'created'):
continue
self.assertEqual(resp.json[key], tale[key])
resp = self.request(
path='/tale/{_id}/export'.format(**tale),
method='GET',
user=self.user,
type='application/octet-stream',
isJson=False)
self.assertStatus(resp, 200)
def path(self, id, type):
user = self.getCurrentUser()
doc = self._getResource(id, type)
if doc is None:
raise RestException('Invalid resource id.')
return path_util.getResourcePath(type, doc, user=user)
def testTaleFlow(self):
from server.lib.license import WholeTaleLicense
resp = self.request(path='/tale',
method='POST',
user=self.user,
type='application/json',
body=json.dumps(
{'imageId': str(self.image['_id'])}))
self.assertStatus(resp, 400)
self.assertEqual(
resp.json, {
'message': ("Invalid JSON object for parameter tale: "
"'dataSet' "
"is a required property"),
'type':
'rest'
})
resp = self.request(path='/tale',
method='POST',
user=self.user,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'dataSet': []
}))
self.assertStatusOk(resp)
tale = resp.json
# Check that workspace was created
# Check that data folder was created
from girder.plugins.wholetale.constants import DATADIRS_NAME
from girder.utility.path import getResourcePath
sc = {
'_id': tale['_id'],
'cname': DATADIRS_NAME,
'fname': DATADIRS_NAME
}
self.assertEqual(
getResourcePath('folder',
Folder().load(tale['folderId'], user=self.user),
user=self.admin),
'/collection/{cname}/{fname}/{_id}'.format(**sc))
taleLicense = WholeTaleLicense.default_spdx()
resp = self.request(path='/tale/{_id}'.format(**tale),
method='PUT',
type='application/json',
user=self.user,
body=json.dumps({
'folderId':
tale['folderId'],
'dataSet':
tale['dataSet'],
'imageId':
tale['imageId'],
'title':
'new name',
'description':
'new description',
'config': {
'memLimit': '2g'
},
'public':
False,
'licenseSPDX':
taleLicense,
'publishInfo': [{
'pid':
'published_pid',
'uri':
'published_url',
'date':
'2019-01-23T15:48:17.476000+00:00',
}]
}))
self.assertStatusOk(resp)
self.assertEqual(resp.json['title'], 'new name')
self.assertEqual(resp.json['licenseSPDX'], taleLicense)
tale = resp.json
resp = self.request(path='/tale',
method='POST',
user=self.user,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'dataSet': [],
}))
self.assertStatusOk(resp)
new_tale = resp.json
resp = self.request(path='/tale',
method='POST',
user=self.admin,
type='application/json',
body=json.dumps({
'imageId': str(self.image['_id']),
'dataSet': [],
'public': False
}))
self.assertStatusOk(resp)
# admin_tale = resp.json
resp = self.request(path='/tale',
method='GET',
user=self.admin,
params={})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 3)
resp = self.request(path='/tale',
method='GET',
user=self.user,
params={'imageId': str(self.image['_id'])})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 2)
self.assertEqual(set([_['_id'] for _ in resp.json]),
{tale['_id'], new_tale['_id']})
resp = self.request(path='/tale',
method='GET',
user=self.user,
params={'userId': str(self.user['_id'])})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 2)
self.assertEqual(set([_['_id'] for _ in resp.json]),
{tale['_id'], new_tale['_id']})
resp = self.request(path='/tale',
method='GET',
user=self.user,
params={'text': 'new'})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
self.assertEqual(set([_['_id'] for _ in resp.json]), {tale['_id']})
resp = self.request(path='/tale/{_id}'.format(**new_tale),
method='DELETE',
user=self.admin)
self.assertStatusOk(resp)
resp = self.request(path='/tale/{_id}'.format(**new_tale),
method='GET',
user=self.user)
self.assertStatus(resp, 400)
resp = self.request(path='/tale/{_id}'.format(**tale),
method='GET',
user=self.user)
self.assertStatusOk(resp)
for key in tale.keys():
if key in ('access', 'updated', 'created'):
continue
self.assertEqual(resp.json[key], tale[key])
def getPathFromRoot(self, user, itemId):
item = self.itemModel.load(itemId, user=user, level=AccessType.READ)
return path_util.getResourcePath('item', item, user=user)
def sync():
import os
from bson.objectid import ObjectId
from girder import logger
from girder.models.assetstore import Assetstore
from girder.models.collection import Collection
from girder.models.file import File
from girder.models.folder import Folder
from girder.models.item import Item
from girder.models.user import User
from girder.exceptions import ResourcePathNotFound
from girder.utility import path as path_utils
from girder.utility import assetstore_utilities
assetstoreId = ObjectId('59b04a2c38eed90001dcc45c')
assetstore = Assetstore().load(id=assetstoreId)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
physicalPath = '/mnt/data/renaissance'
collectionPath = '/collection/Renaissance Simulations'
collectionId = ObjectId('59b04a0e38eed90001dcc45b')
rslCollection = Collection().load(collectionId, force=True)
admin = list(User().getAdmins())[0]
def purge_leaf_folder(path):
folder = path_utils.lookUpPath(path, user=admin)['document']
if Item().find({'folderId': folder['_id']}).count() > 0 or \
list(Folder().childFolders(folder, 'folder', user=admin)):
return
logger.info("Removing empty folder %s" % path)
Folder().remove(folder)
purge_leaf_folder(os.path.dirname(path))
q = {'assetstoreId': assetstoreId, 'imported': True}
fields = ['path', 'size', 'name', 'mtime']
girderFiles = {
fObj.pop('path'): fObj
for fObj in File().find(q, fields=fields)
}
toImport = {}
toModify = {}
for (dirpath, dirnames, filenames) in os.walk(physicalPath):
for filename in filenames:
path = os.path.join(dirpath, filename)
stat = os.stat(path)
try:
girderObj = girderFiles.pop(path)
except KeyError:
toImport[path] = {
'mtime': stat.st_mtime,
'size': stat.st_size,
'name': filename
}
if girderObj['mtime'] != stat.st_mtime or \
girderObj['size'] != stat.st_size or \
girderObj['name'] != filename:
girderObj.update({
'mtime': stat.st_mtime,
'size': stat.st_size,
'name': filename,
'path': path
})
toModify[girderObj.pop('_id')] = girderObj
# Remove orphaned files
potentialLeafFolders = set()
for orphan in girderFiles.values():
fileObj = File().load(ObjectId(orphan['_id']), force=True)
itemObj = Item().load(fileObj['itemId'], force=True)
File().remove(fileObj)
girderPath = path_utils.getResourcePath('item', itemObj, force=True)
if not list(Item().childFiles(itemObj)):
Item().remove(itemObj)
potentialLeafFolders.add(os.path.dirname(girderPath))
logger.info('Removed %s' % girderPath)
# Remove empty folders
for path in list(potentialLeafFolders):
purge_leaf_folder(path)
# Import new items
for filePath, newFile in toImport.items():
relpath = os.path.relpath(filePath, physicalPath)
parentType = 'collection'
parent = rslCollection
dirs = os.path.dirname(relpath).split('/')
for directory in dirs:
try:
parent, parentType = \
path_utils.lookUpToken(directory, parentType, parent)
except ResourcePathNotFound:
parent = Folder().createFolder(parent,
directory,
parentType=parentType,
public=True,
creator=admin)
parentType = 'folder'
adapter._importDataAsItem(os.path.basename(relpath),
admin,
parent,
os.path.dirname(filePath),
[os.path.basename(filePath)],
reuseExisting=True)
logger.info('Imported %s to %s' %
(filePath, os.path.join(collectionPath, relpath)))
