def importData(self, parent, parentType, params, progress, user,
leafFoldersAsItems):
importPath = params['importPath']
if not os.path.exists(importPath):
raise ValidationException('Not found: %s.' % importPath)
if not os.path.isdir(importPath):
name = os.path.basename(importPath)
progress.update(message=name)
self._importFileToFolder(name, user, parent, parentType,
importPath)
return
listDir = os.listdir(importPath)
if leafFoldersAsItems and self._hasOnlyFiles(importPath, listDir):
self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)),
user,
parent,
importPath,
listDir,
params=params)
return
for name in listDir:
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
localListDir = os.listdir(path)
if leafFoldersAsItems and self._hasOnlyFiles(
path, localListDir):
self._importDataAsItem(name,
user,
parent,
path,
localListDir,
params=params)
else:
folder = self.model('folder').createFolder(
parent=parent,
name=name,
parentType=parentType,
creator=user,
reuseExisting=True)
events.trigger('filesystem_assetstore_imported', {
'id': folder['_id'],
'type': 'folder',
'importPath': path
})
nextPath = os.path.join(importPath, name)
self.importData(folder,
'folder',
params=dict(params, importPath=nextPath),
progress=progress,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
else:
if self.shouldImportFile(path, params):
self._importFileToFolder(name, user, parent, parentType,
path)
def importData(self, parent, parentType, params, progress, user):
importPath = params['importPath']
if not os.path.isdir(importPath):
raise ValidationException('No such directory: %s.' % importPath)
for name in os.listdir(importPath):
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
folder = self.model('folder').createFolder(
parent=parent, name=name, parentType=parentType,
creator=user, reuseExisting=True)
self.importData(folder, 'folder', params={
'importPath': os.path.join(importPath, name)
}, progress=progress, user=user)
else:
if parentType != 'folder':
raise ValidationException(
'Files cannot be imported directly underneath a %s.' %
parentType)
item = self.model('item').createItem(
name=name, creator=user, folder=parent, reuseExisting=True)
self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user):
importPath = params['importPath']
if not os.path.isdir(importPath):
raise ValidationException('No such directory: %s.' % importPath)
for name in os.listdir(importPath):
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
folder = self.model('folder').createFolder(
parent=parent,
name=name,
parentType=parentType,
creator=user,
reuseExisting=True)
self.importData(
folder,
'folder',
params={'importPath': os.path.join(importPath, name)},
progress=progress,
user=user)
else:
if parentType != 'folder':
raise ValidationException(
'Files cannot be imported directly underneath a %s.' %
parentType)
item = self.model('item').createItem(name=name,
creator=user,
folder=parent,
reuseExisting=True)
self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user,
leafFoldersAsItems):
importPath = params['importPath']
if not os.path.isdir(importPath):
raise ValidationException('No such directory: %s.' % importPath)
def hasOnlyFiles(path, files):
return all(
map(os.path.isfile,
(os.path.join(path, fname) for fname in files)))
listDir = os.listdir(importPath)
if leafFoldersAsItems and hasOnlyFiles(importPath, listDir):
self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)),
user, parent, importPath, listDir)
return
for name in listDir:
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
localListDir = os.listdir(path)
if leafFoldersAsItems and hasOnlyFiles(path, localListDir):
self._importDataAsItem(name, user, parent, path,
localListDir)
else:
folder = self.model('folder').createFolder(
parent=parent,
name=name,
parentType=parentType,
creator=user,
reuseExisting=True)
self.importData(
folder,
'folder',
params={'importPath': os.path.join(importPath, name)},
progress=progress,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
else:
if parentType != 'folder':
raise ValidationException(
'Files cannot be imported directly'
'underneath a %s.' % parentType)
item = self.model('item').createItem(name=name,
creator=user,
folder=parent,
reuseExisting=True)
self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user, leafFoldersAsItems):
importPath = params['importPath']
if not os.path.exists(importPath):
raise ValidationException('Not found: %s.' % importPath)
if not os.path.isdir(importPath):
name = os.path.basename(importPath)
progress.update(message=name)
self._importFileToFolder(name, user, parent, parentType, importPath)
return
listDir = os.listdir(importPath)
if parentType != 'folder' and any(
os.path.isfile(os.path.join(importPath, val)) for val in listDir):
raise ValidationException(
'Files cannot be imported directly underneath a %s.' % parentType)
if leafFoldersAsItems and self._hasOnlyFiles(importPath, listDir):
self._importDataAsItem(
os.path.basename(importPath.rstrip(os.sep)), user, parent, importPath,
listDir, params=params)
return
for name in listDir:
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
localListDir = os.listdir(path)
if leafFoldersAsItems and self._hasOnlyFiles(path, localListDir):
self._importDataAsItem(name, user, parent, path, localListDir, params=params)
else:
folder = Folder().createFolder(
parent=parent, name=name, parentType=parentType,
creator=user, reuseExisting=True)
events.trigger(
'filesystem_assetstore_imported', {
'id': folder['_id'],
'type': 'folder',
'importPath': path
})
nextPath = os.path.join(importPath, name)
self.importData(
folder, 'folder', params=dict(params, importPath=nextPath),
progress=progress, user=user, leafFoldersAsItems=leafFoldersAsItems)
else:
if self.shouldImportFile(path, params):
self._importFileToFolder(name, user, parent, parentType, path)
def importData(self, parent, parentType, params, progress, user,
leafFoldersAsItems):
importPath = params['importPath']
if not os.path.isdir(importPath):
raise ValidationException('No such directory: %s.' % importPath)
def hasOnlyFiles(path, files):
return all(map(os.path.isfile, (os.path.join(path, fname)
for fname in files)))
listDir = os.listdir(importPath)
if leafFoldersAsItems and hasOnlyFiles(importPath, listDir):
self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)),
user, parent, importPath, listDir)
return
for name in listDir:
progress.update(message=name)
path = os.path.join(importPath, name)
if os.path.isdir(path):
localListDir = os.listdir(path)
if leafFoldersAsItems and hasOnlyFiles(path, localListDir):
self._importDataAsItem(name, user, parent, path,
localListDir)
else:
folder = self.model('folder').createFolder(
parent=parent, name=name, parentType=parentType,
creator=user, reuseExisting=True)
self.importData(folder, 'folder', params={
'importPath': os.path.join(importPath, name)},
progress=progress, user=user,
leafFoldersAsItems=leafFoldersAsItems)
else:
if parentType != 'folder':
raise ValidationException(
'Files cannot be imported directly'
'underneath a %s.' %
parentType)
item = self.model('item').createItem(
name=name, creator=user, folder=parent,
reuseExisting=True)
self.importFile(item, path, user, name=name)
def findInvalidFiles(self, progress=progress.noProgress, filters=None,
checkSize=True, **kwargs):
"""
Goes through every file in this assetstore and finds those whose
underlying data is missing or invalid. This is a generator function --
for each invalid file found, a dictionary is yielded to the caller that
contains the file, its absolute path on disk, and a reason for invalid,
e.g. "missing" or "size".
:param progress: Pass a progress context to record progress.
:type progress: :py:class:`girder.utility.progress.ProgressContext`
:param filters: Additional query dictionary to restrict the search for
files. There is no need to set the ``assetstoreId`` in the filters,
since that is done automatically.
:type filters: dict or None
:param checkSize: Whether to make sure the size of the underlying
data matches the size of the file.
:type checkSize: bool
"""
filters = filters or {}
q = dict({
'assetstoreId': self.assetstore['_id']
}, **filters)
cursor = File().find(q)
progress.update(total=cursor.count(), current=0)
for file in cursor:
progress.update(increment=1, message=file['name'])
path = self.fullPath(file)
if not os.path.isfile(path):
yield {
'reason': 'missing',
'file': file,
'path': path
}
elif checkSize and os.path.getsize(path) != file['size']:
yield {
'reason': 'size',
'file': file,
'path': path
}
def findInvalidFiles(self,
progress=progress.noProgress,
filters=None,
checkSize=True,
**kwargs):
"""
Goes through every file in this assetstore and finds those whose
underlying data is missing or invalid. This is a generator function --
for each invalid file found, a dictionary is yielded to the caller that
contains the file, its absolute path on disk, and a reason for invalid,
e.g. "missing" or "size".
:param progress: Pass a progress context to record progress.
:type progress: :py:class:`girder.utility.progress.ProgressContext`
:param filters: Additional query dictionary to restrict the search for
files. There is no need to set the ``assetstoreId`` in the filters,
since that is done automatically.
:type filters: dict or None
:param checkSize: Whether to make sure the size of the underlying
data matches the size of the file.
:type checkSize: bool
"""
filters = filters or {}
q = dict({'assetstoreId': self.assetstore['_id']}, **filters)
cursor = File().find(q)
progress.update(total=cursor.count(), current=0)
for file in cursor:
progress.update(increment=1, message=file['name'])
path = self.fullPath(file)
if not os.path.isfile(path):
yield {'reason': 'missing', 'file': file, 'path': path}
elif checkSize and os.path.getsize(path) != file['size']:
yield {'reason': 'size', 'file': file, 'path': path}
