def isOrphan(self, file):
"""
Returns True if this file is orphaned (its item or attached entity is
missing).
:param file: The file to check.
:type file: dict
"""
if file.get('attachedToId'):
attachedToType = file.get('attachedToType')
if isinstance(attachedToType, six.string_types):
modelType = ModelImporter.model(attachedToType)
elif isinstance(attachedToType, list) and len(attachedToType) == 2:
modelType = ModelImporter.model(*attachedToType)
else:
# Invalid 'attachedToType'
return True
if isinstance(
modelType,
(acl_mixin.AccessControlMixin, AccessControlledModel)):
attachedDoc = modelType.load(file.get('attachedToId'),
force=True)
else:
attachedDoc = modelType.load(file.get('attachedToId'))
else:
from .item import Item
attachedDoc = Item().load(file.get('itemId'), force=True)
return not attachedDoc
def _commonSearchModeHandler(mode, query, types, user, level, limit, offset):
"""
The common handler for `text` and `prefix` search modes.
"""
# Avoid circular import
from girderformindlogger.utility.resource import allowedSearchTypes
method = '%sSearch' % mode
results = {}
for modelName in types:
if modelName not in allowedSearchTypes:
continue
if '.' in modelName:
name, plugin = modelName.rsplit('.', 1)
model = ModelImporter.model(name, plugin)
else:
model = ModelImporter.model(modelName)
if model is not None:
results[modelName] = [
model.filter(d, user) for d in getattr(model, method)(
query=query, user=user, limit=limit, offset=offset, level=level)
]
return results
def parentsToRoot(self, folder, curPath=None, user=None, force=False, level=AccessType.READ):
"""
Get the path to traverse to a root of the hierarchy.
:param folder: The folder whose root to find
:type folder: dict
:returns: an ordered list of dictionaries from root to the current folder
"""
curPath = curPath or []
curParentId = folder['parentId']
curParentType = folder['parentCollection']
if curParentType in ('user', 'collection'):
curParentObject = ModelImporter.model(curParentType).load(
curParentId, user=user, level=level, force=force)
if force:
parentFiltered = curParentObject
else:
parentFiltered = ModelImporter.model(curParentType).filter(curParentObject, user)
return [{
'type': curParentType,
'object': parentFiltered
}] + curPath
else:
curParentObject = self.load(curParentId, user=user, level=level, force=force)
curPath = [{
'type': curParentType,
'object': curParentObject if force else self.filter(curParentObject, user)
}] + curPath
return self.parentsToRoot(curParentObject, curPath, user=user, force=force)
def propagateSizeChange(self, item, inc):
from girderformindlogger.models.folder import Folder
Folder().increment(query={'_id': item['folderId']},
field='size',
amount=inc,
multi=False)
ModelImporter.model(item['baseParentType']).increment(
query={'_id': item['baseParentId']},
field='size',
amount=inc,
multi=False)
def createThumbnail(self, file, width, height, crop, attachToId, attachToType):
user = self.getCurrentUser()
ModelImporter.model(attachToType).load(
attachToId, user=user, level=AccessType.WRITE, exc=True)
width = max(width, 0)
height = max(height, 0)
if not width and not height:
raise RestException('You must specify a valid width, height, or both.')
return utils.scheduleThumbnailJob(file, attachToType, attachToId, user, width, height, crop)
def _getAssetstoreModel(self, file):
from .assetstore import Assetstore
if file.get('assetstoreType'):
try:
if isinstance(file['assetstoreType'], six.string_types):
return ModelImporter.model(file['assetstoreType'])
else:
return ModelImporter.model(*file['assetstoreType'])
except Exception:
raise ValidationException('Invalid assetstore type: %s.' %
(file['assetstoreType'], ))
else:
return Assetstore()
def _setResourceQuota(self, model, resource, policy):
"""
Handle setting quota policies for any resource that supports them.
:param model: the type of resource (e.g., user or collection)
:param resource: the resource document.
:param params: the query parameters. 'policy' is required and used.
:returns: the updated resource document.
"""
policy = self._validatePolicy(policy)
if QUOTA_FIELD not in resource:
resource[QUOTA_FIELD] = {}
resource[QUOTA_FIELD].update(policy)
ModelImporter.model(model).save(resource, validate=False)
return self._filter(model, resource)
def find(self, parentType, parentId, text, name, limit, offset, sort):
"""
Get a list of folders with given search parameters. Currently accepted
search modes are:
1. Searching by parentId and parentType, with optional additional
filtering by the name field (exact match) or using full text search
within a single parent folder. Pass a "name" parameter or "text"
parameter to invoke these additional filters.
2. Searching with full text search across all folders in the system.
Simply pass a "text" parameter for this mode.
"""
user = self.getCurrentUser()
if parentType and parentId:
parent = ModelImporter.model(parentType).load(
parentId, user=user, level=AccessType.READ, exc=True)
filters = {}
if text:
filters['$text'] = {
'$search': text
}
if name:
filters['name'] = name
return self._model.childFolders(
parentType=parentType, parent=parent, user=user,
offset=offset, limit=limit, sort=sort, filters=filters)
elif text:
return self._model.textSearch(
text, user=user, limit=limit, offset=offset, sort=sort)
else:
raise RestException('Invalid search mode.')
def wrapped(*args, **kwargs):
model = ModelImporter.model(self.model, self.plugin)
for raw, converted in six.viewitems(self.map):
id = self._getIdValue(kwargs, raw)
if self.force:
kwargs[converted] = model.load(
id, force=True, **self.kwargs)
elif self.level is not None:
kwargs[converted] = model.load(
id=id, level=self.level, user=getCurrentUser(),
**self.kwargs)
else:
kwargs[converted] = model.load(id, **self.kwargs)
if kwargs[converted] is None and self.exc:
raise RestException(
'Invalid %s id (%s).' % (model.name, str(id)))
if self.requiredFlags:
model.requireAccessFlags(
kwargs[converted], user=getCurrentUser(), flags=self.requiredFlags)
return fun(*args, **kwargs)
def copyFolder(self, srcFolder, parent=None, name=None, description=None,
parentType=None, public=None, creator=None, progress=None,
firstFolder=None):
"""
Copy a folder, including all child items and child folders.
:param srcFolder: the folder to copy.
:type srcFolder: dict
:param parent: The parent document. Must be a folder, user, or
collection.
:type parent: dict
:param name: The name of the new folder. None to copy the original
name.
:type name: str
:param description: Description for the new folder. None to copy the
original description.
:type description: str
:param parentType: What type the parent is:
('folder' | 'user' | 'collection')
:type parentType: str
:param public: Public read access flag. None to inherit from parent,
'original' to inherit from original folder.
:type public: bool, None, or 'original'.
:param creator: user representing the creator of the new folder.
:type creator: dict
:param progress: a progress context to record process on.
:type progress: girderformindlogger.utility.progress.ProgressContext or None.
:param firstFolder: if not None, the first folder copied in a tree of
folders.
:returns: the new folder document.
"""
setResponseTimeLimit()
if parentType is None:
parentType = srcFolder['parentCollection']
parentType = parentType.lower()
if parentType not in ('folder', 'user', 'collection'):
raise ValidationException('The parentType must be folder, '
'collection, or user.')
if parent is None:
parent = ModelImporter.model(parentType).load(srcFolder['parentId'], force=True)
if name is None:
name = srcFolder['name']
if description is None:
description = srcFolder['description']
if public is not None and isinstance(public, six.string_types):
if public == 'original':
public = srcFolder.get('public', None)
else:
public = public == 'true'
newFolder = self.createFolder(
parentType=parentType, parent=parent, name=name,
description=description, public=public, creator=creator,
allowRename=True)
if firstFolder is None:
firstFolder = newFolder
return self.copyFolderComponents(
srcFolder, newFolder, creator, progress, firstFolder)
def isOrphan(self, folder):
"""
Returns True if this folder is orphaned (its parent is missing).
:param folder: The folder to check.
:type folder: dict
"""
return not ModelImporter.model(folder.get('parentCollection')).load(
folder.get('parentId'), force=True)
def _getModel(self, name, modelParams):
if name not in self.description.modelParams:
return
info = self.description.modelParams[name]
if info['isModelClass']:
return info['model']()
else:
return ModelImporter.model(info['model'], info['plugin'])
def _prepareMoveOrCopy(self, resources, parentType, parentId):
user = self.getCurrentUser()
self._validateResourceSet(resources, ('folder', 'item'))
if resources.get('item') and parentType != 'folder':
raise RestException('Invalid parentType.')
return ModelImporter.model(parentType).load(parentId,
level=AccessType.WRITE,
user=user,
exc=True)
def _getBaseResource(self, model, resource):
"""
Get the base resource for something pertaining to quota policies. If
the base resource has no quota policy, return (None, None).
:param model: the initial model type. Could be file, item, folder,
user, or collection.
:param resource: the initial resource document.
:returns: A pair ('model', 'resource'), where 'model' is the base model
type, either 'user' or 'collection'., and 'resource' is the
base resource document or the id of that document.
"""
if isinstance(resource, six.string_types + (ObjectId, )):
try:
resource = ModelImporter.model(model).load(id=resource,
force=True)
except ImportError:
return None, None
if model == 'file':
model = 'item'
resource = Item().load(id=resource['itemId'], force=True)
if model in ('folder', 'item'):
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
resource = ModelImporter.model(model).load(id=resource['_id'],
force=True)
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
return None, None
model = resource['baseParentType']
resourceId = resource['baseParentId']
resource = ModelImporter.model(model).load(id=resourceId,
force=True)
if model in ('user', 'collection') and resource:
# Ensure the base resource has a quota field so we can use the
# default quota if appropriate
if QUOTA_FIELD not in resource:
resource[QUOTA_FIELD] = {}
if not resource or QUOTA_FIELD not in resource:
return None, None
return model, resource
def createFolder(self, public, parentType, parentId, name, description,
reuseExisting, metadata):
user = self.getCurrentUser()
parent = ModelImporter.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
newFolder = self._model.createFolder(
parent=parent, name=name, parentType=parentType, creator=user,
description=description, public=public, reuseExisting=reuseExisting)
if metadata:
newFolder = self._model.setMetadata(newFolder, metadata)
return newFolder
def _filter(self, model, resource):
"""
Filter a resource to include only the ordinary data and the quota
field.
:param model: the type of resource (e.g., user or collection)
:param resource: the resource document.
:returns: filtered field of the resource with the quota data, if any.
"""
filtered = ModelImporter.model(model).filter(resource,
self.getCurrentUser())
filtered[QUOTA_FIELD] = resource.get(QUOTA_FIELD, {})
return filtered
def propagateSizeChange(self, item, sizeIncrement, updateItemSize=True):
"""
Propagates a file size change (or file creation) to the necessary
parents in the hierarchy. Internally, this records subtree size in
the item, the parent folder, and the root node under which the item
lives. Should be called anytime a new file is added, a file is
deleted, or a file size changes.
:param item: The parent item of the file.
:type item: dict
:param sizeIncrement: The change in size to propagate.
:type sizeIncrement: int
:param updateItemSize: Whether the item size should be updated. Set to
False if you plan to delete the item immediately and don't care to
update its size.
"""
from .folder import Folder
from .item import Item
if updateItemSize:
# Propagate size up to item
Item().increment(query={'_id': item['_id']},
field='size',
amount=sizeIncrement,
multi=False)
# Propagate size to direct parent folder
Folder().increment(query={'_id': item['folderId']},
field='size',
amount=sizeIncrement,
multi=False)
# Propagate size up to root data node
ModelImporter.model(item['baseParentType']).increment(
query={'_id': item['baseParentId']},
field='size',
amount=sizeIncrement,
multi=False)
def removeThumbnailLink(event):
"""
When a thumbnail file is deleted, we remove the reference to it from the
resource to which it is attached.
"""
doc = event.info
if doc.get('isThumbnail'):
model = ModelImporter.model(doc['attachedToType'])
resource = model.load(doc['attachedToId'], force=True)
if doc['_id'] in resource.get('_thumbnails', ()):
resource['_thumbnails'].remove(doc['_id'])
model.save(resource, validate=False)
def stream():
zip = ziputil.ZipGenerator()
for kind in resources:
model = ModelImporter.model(kind)
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.READ)
for (path, file) in model.fileList(
doc=doc,
user=user,
includeMetadata=includeMetadata,
subpath=True):
for data in zip.addFile(file, path):
yield data
yield zip.footer()
def _getResourceModel(self, kind, funcName=None):
"""
Load and return a model with a specific function or throw an exception.
:param kind: the name of the model to load
:param funcName: a function name to ensure that each model contains.
:returns: the loaded model.
"""
try:
model = ModelImporter.model(kind)
except Exception:
model = None
if not model or (funcName and not hasattr(model, funcName)):
raise RestException('Invalid resources format.')
return model
def copyFolder(self, folder, parentType, parentId, name, description, public, progress):
user = self.getCurrentUser()
parentType = parentType or folder['parentCollection']
if parentId:
parent = ModelImporter.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
else:
parent = None
with ProgressContext(progress, user=self.getCurrentUser(),
title='Copying folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder))
return self._model.copyFolder(
folder, creator=user, name=name, parentType=parentType,
parent=parent, description=description, public=public, progress=ctx)
def updateFolder(self, folder, name, description, parentType, parentId, metadata):
user = self.getCurrentUser()
if name is not None:
folder['name'] = name
if description is not None:
folder['description'] = description
folder = self._model.updateFolder(folder)
if metadata:
folder = self._model.setMetadata(folder, metadata)
if parentType and parentId:
parent = ModelImporter.model(parentType).load(
parentId, level=AccessType.WRITE, user=user, exc=True)
if (parentType, parent['_id']) != (folder['parentCollection'], folder['parentId']):
folder = self._model.move(folder, parent, parentType)
return folder
def list_folder(self, path):
path = path.rstrip('/')
entries = []
if path == '':
for model in ('collection', 'user'):
info = paramiko.SFTPAttributes()
info.st_size = 0
info.st_mode = 0o777 | stat.S_IFDIR
info.filename = model.encode('utf8')
entries.append(info)
elif path in ('/user', '/collection'):
model = path[1:]
for doc in ModelImporter.model(model).list(user=self.server.girderUser):
entries.append(_stat(doc, model))
else:
obj = lookUpPath(path, filter=False, user=self.server.girderUser)
return self._list(obj['model'], obj['document'])
return entries
def importData(self, assetstore, importPath, destinationId,
destinationType, progress, leafFoldersAsItems,
fileIncludeRegex, fileExcludeRegex):
user = self.getCurrentUser()
parent = ModelImporter.model(destinationType).load(
destinationId, user=user, level=AccessType.ADMIN, exc=True)
with ProgressContext(progress, user=user,
title='Importing data') as ctx:
return self._model.importData(
assetstore,
parent=parent,
parentType=destinationType,
params={
'fileIncludeRegex': fileIncludeRegex,
'fileExcludeRegex': fileExcludeRegex,
'importPath': importPath,
},
progress=ctx,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
def readdir(self, path, fh):
"""
Get a list of names within a directory.
:param path: path within the fuse.
:param fh: an open file handle. Ignored, since path is always
specified.
:returns: a list of names. This always includes . and ..
"""
path = path.rstrip('/')
result = [u'.', u'..']
if path == '':
result.extend([u'collection', u'user'])
elif path in ('/user', '/collection'):
model = path[1:]
docList = ModelImporter.model(model).find({}, sort=None)
for doc in docList:
result.append(self._name(doc, model))
else:
resource = self._getPath(path)
result.extend(self._list(resource['document'], resource['model']))
return result
def wrapped(*args, **kwargs):
val = fun(*args, **kwargs)
if val is None:
return None
if self._isModelClass:
model = self.model()
else:
model = ModelImporter.model(self.model, self.plugin)
user = getCurrentUser()
if isinstance(val, _MONGO_CURSOR_TYPES):
if callable(getattr(val, 'count', None)):
cherrypy.response.headers['Girder-Total-Count'] = val.count()
return [model.filter(m, user, self.addFields) for m in val]
elif isinstance(val, (list, tuple, types.GeneratorType)):
return [model.filter(m, user, self.addFields) for m in val]
elif isinstance(val, dict):
return model.filter(val, user, self.addFields)
else:
raise Exception('Cannot call filtermodel on return type: %s.' % type(val))
def attachThumbnail(file, thumbnail, attachToType, attachToId, width, height):
"""
Add the required information to the thumbnail file and the resource it
is being attached to, and save the documents.
:param file: The file from which the thumbnail was derived.
:type file: dict
:param thumbnail: The newly generated thumbnail file document.
:type thumbnail: dict
:param attachToType: The type to which the thumbnail is being attached.
:type attachToType: str
:param attachToId: The ID of the document to attach the thumbnail to.
:type attachToId: str or ObjectId
:param width: Thumbnail width.
:type width: int
:param height: Thumbnail height.
:type height: int
:returns: The updated thumbnail file document.
"""
parentModel = ModelImporter.model(attachToType)
parent = parentModel.load(attachToId, force=True)
parent['_thumbnails'] = parent.get('_thumbnails', [])
parent['_thumbnails'].append(thumbnail['_id'])
parentModel.save(parent)
thumbnail['attachedToType'] = attachToType
thumbnail['attachedToId'] = parent['_id']
thumbnail['isThumbnail'] = True
thumbnail['derivedFrom'] = {
'type': 'file',
'id': file['_id'],
'process': 'thumbnail',
'width': width,
'height': height
}
return File().save(thumbnail)
def propagateSizeChange(folder, inc):
ModelImporter.model(folder['baseParentType']).increment(query={
'_id': folder['baseParentId']
}, field='size', amount=inc, multi=False)
def load(self, info):
ModelImporter.registerModel('job', Job, 'jobs')
info['apiRoot'].job = job_rest.Job()
events.bind('jobs.schedule', 'jobs', scheduleLocal)
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl,
reference, assetstoreId):
"""
Before any bytes of the actual file are sent, a request should be made
to initialize the upload. This creates the temporary record of the
forthcoming upload that will be passed in chunks to the readChunk
method. If you pass a "linkUrl" parameter, it will make a link file
in the designated parent.
"""
user = self.getCurrentUser()
parent = ModelImporter.model(parentType).load(id=parentId,
user=user,
level=AccessType.WRITE,
exc=True)
if linkUrl is not None:
return self._model.filter(
self._model.createLinkFile(url=linkUrl,
parent=parent,
name=name,
parentType=parentType,
creator=user,
size=size,
mimeType=mimeType), user)
else:
self.requireParams({'size': size})
assetstore = None
if assetstoreId:
self.requireAdmin(
user,
message=
'You must be an admin to select a destination assetstore.')
assetstore = Assetstore().load(assetstoreId)
chunk = None
if size > 0 and cherrypy.request.headers.get('Content-Length'):
ct = cherrypy.request.body.content_type.value
if (ct not in cherrypy.request.body.processors and ct.split(
'/', 1)[0] not in cherrypy.request.body.processors):
chunk = RequestBodyStream(cherrypy.request.body)
if chunk is not None and chunk.getSize() <= 0:
chunk = None
try:
# TODO: This can be made more efficient by adding
# save=chunk is None
# to the createUpload call parameters. However, since this is
# a breaking change, that should be deferred until a major
# version upgrade.
upload = Upload().createUpload(user=user,
name=name,
parentType=parentType,
parent=parent,
size=size,
mimeType=mimeType,
reference=reference,
assetstore=assetstore)
except OSError as exc:
if exc.errno == errno.EACCES:
raise GirderException(
'Failed to create upload.',
'girderformindlogger.api.v1.file.create-upload-failed')
raise
if upload['size'] > 0:
if chunk:
return Upload().handleChunk(upload,
chunk,
filter=True,
user=user)
return upload
else:
return self._model.filter(Upload().finalizeUpload(upload),
user)