def importData(self, parent, parentType, params, progress, user, leafFoldersAsItems): importPath = params['importPath'] if not os.path.exists(importPath): raise ValidationException('Not found: %s.' % importPath) if not os.path.isdir(importPath): name = os.path.basename(importPath) progress.update(message=name) self._importFileToFolder(name, user, parent, parentType, importPath) return listDir = os.listdir(importPath) if leafFoldersAsItems and self._hasOnlyFiles(importPath, listDir): self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)), user, parent, importPath, listDir, params=params) return for name in listDir: progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): localListDir = os.listdir(path) if leafFoldersAsItems and self._hasOnlyFiles( path, localListDir): self._importDataAsItem(name, user, parent, path, localListDir, params=params) else: folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) events.trigger('filesystem_assetstore_imported', { 'id': folder['_id'], 'type': 'folder', 'importPath': path }) nextPath = os.path.join(importPath, name) self.importData(folder, 'folder', params=dict(params, importPath=nextPath), progress=progress, user=user, leafFoldersAsItems=leafFoldersAsItems) else: if self.shouldImportFile(path, params): self._importFileToFolder(name, user, parent, parentType, path)
def importData(self, parent, parentType, params, progress, user): importPath = params['importPath'] if not os.path.isdir(importPath): raise ValidationException('No such directory: %s.' % importPath) for name in os.listdir(importPath): progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) self.importData(folder, 'folder', params={ 'importPath': os.path.join(importPath, name) }, progress=progress, user=user) else: if parentType != 'folder': raise ValidationException( 'Files cannot be imported directly underneath a %s.' % parentType) item = self.model('item').createItem( name=name, creator=user, folder=parent, reuseExisting=True) self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user): importPath = params['importPath'] if not os.path.isdir(importPath): raise ValidationException('No such directory: %s.' % importPath) for name in os.listdir(importPath): progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) self.importData( folder, 'folder', params={'importPath': os.path.join(importPath, name)}, progress=progress, user=user) else: if parentType != 'folder': raise ValidationException( 'Files cannot be imported directly underneath a %s.' % parentType) item = self.model('item').createItem(name=name, creator=user, folder=parent, reuseExisting=True) self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user, leafFoldersAsItems): importPath = params['importPath'] if not os.path.isdir(importPath): raise ValidationException('No such directory: %s.' % importPath) def hasOnlyFiles(path, files): return all( map(os.path.isfile, (os.path.join(path, fname) for fname in files))) listDir = os.listdir(importPath) if leafFoldersAsItems and hasOnlyFiles(importPath, listDir): self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)), user, parent, importPath, listDir) return for name in listDir: progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): localListDir = os.listdir(path) if leafFoldersAsItems and hasOnlyFiles(path, localListDir): self._importDataAsItem(name, user, parent, path, localListDir) else: folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) self.importData( folder, 'folder', params={'importPath': os.path.join(importPath, name)}, progress=progress, user=user, leafFoldersAsItems=leafFoldersAsItems) else: if parentType != 'folder': raise ValidationException( 'Files cannot be imported directly' 'underneath a %s.' % parentType) item = self.model('item').createItem(name=name, creator=user, folder=parent, reuseExisting=True) self.importFile(item, path, user, name=name)
def importData(self, parent, parentType, params, progress, user, leafFoldersAsItems): importPath = params['importPath'] if not os.path.exists(importPath): raise ValidationException('Not found: %s.' % importPath) if not os.path.isdir(importPath): name = os.path.basename(importPath) progress.update(message=name) self._importFileToFolder(name, user, parent, parentType, importPath) return listDir = os.listdir(importPath) if parentType != 'folder' and any( os.path.isfile(os.path.join(importPath, val)) for val in listDir): raise ValidationException( 'Files cannot be imported directly underneath a %s.' % parentType) if leafFoldersAsItems and self._hasOnlyFiles(importPath, listDir): self._importDataAsItem( os.path.basename(importPath.rstrip(os.sep)), user, parent, importPath, listDir, params=params) return for name in listDir: progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): localListDir = os.listdir(path) if leafFoldersAsItems and self._hasOnlyFiles(path, localListDir): self._importDataAsItem(name, user, parent, path, localListDir, params=params) else: folder = Folder().createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) events.trigger( 'filesystem_assetstore_imported', { 'id': folder['_id'], 'type': 'folder', 'importPath': path }) nextPath = os.path.join(importPath, name) self.importData( folder, 'folder', params=dict(params, importPath=nextPath), progress=progress, user=user, leafFoldersAsItems=leafFoldersAsItems) else: if self.shouldImportFile(path, params): self._importFileToFolder(name, user, parent, parentType, path)
def importData(self, parent, parentType, params, progress, user, leafFoldersAsItems): importPath = params['importPath'] if not os.path.isdir(importPath): raise ValidationException('No such directory: %s.' % importPath) def hasOnlyFiles(path, files): return all(map(os.path.isfile, (os.path.join(path, fname) for fname in files))) listDir = os.listdir(importPath) if leafFoldersAsItems and hasOnlyFiles(importPath, listDir): self._importDataAsItem(os.path.basename(importPath.rstrip(os.sep)), user, parent, importPath, listDir) return for name in listDir: progress.update(message=name) path = os.path.join(importPath, name) if os.path.isdir(path): localListDir = os.listdir(path) if leafFoldersAsItems and hasOnlyFiles(path, localListDir): self._importDataAsItem(name, user, parent, path, localListDir) else: folder = self.model('folder').createFolder( parent=parent, name=name, parentType=parentType, creator=user, reuseExisting=True) self.importData(folder, 'folder', params={ 'importPath': os.path.join(importPath, name)}, progress=progress, user=user, leafFoldersAsItems=leafFoldersAsItems) else: if parentType != 'folder': raise ValidationException( 'Files cannot be imported directly' 'underneath a %s.' % parentType) item = self.model('item').createItem( name=name, creator=user, folder=parent, reuseExisting=True) self.importFile(item, path, user, name=name)
def findInvalidFiles(self, progress=progress.noProgress, filters=None, checkSize=True, **kwargs): """ Goes through every file in this assetstore and finds those whose underlying data is missing or invalid. This is a generator function -- for each invalid file found, a dictionary is yielded to the caller that contains the file, its absolute path on disk, and a reason for invalid, e.g. "missing" or "size". :param progress: Pass a progress context to record progress. :type progress: :py:class:`girder.utility.progress.ProgressContext` :param filters: Additional query dictionary to restrict the search for files. There is no need to set the ``assetstoreId`` in the filters, since that is done automatically. :type filters: dict or None :param checkSize: Whether to make sure the size of the underlying data matches the size of the file. :type checkSize: bool """ filters = filters or {} q = dict({ 'assetstoreId': self.assetstore['_id'] }, **filters) cursor = File().find(q) progress.update(total=cursor.count(), current=0) for file in cursor: progress.update(increment=1, message=file['name']) path = self.fullPath(file) if not os.path.isfile(path): yield { 'reason': 'missing', 'file': file, 'path': path } elif checkSize and os.path.getsize(path) != file['size']: yield { 'reason': 'size', 'file': file, 'path': path }
def findInvalidFiles(self, progress=progress.noProgress, filters=None, checkSize=True, **kwargs): """ Goes through every file in this assetstore and finds those whose underlying data is missing or invalid. This is a generator function -- for each invalid file found, a dictionary is yielded to the caller that contains the file, its absolute path on disk, and a reason for invalid, e.g. "missing" or "size". :param progress: Pass a progress context to record progress. :type progress: :py:class:`girder.utility.progress.ProgressContext` :param filters: Additional query dictionary to restrict the search for files. There is no need to set the ``assetstoreId`` in the filters, since that is done automatically. :type filters: dict or None :param checkSize: Whether to make sure the size of the underlying data matches the size of the file. :type checkSize: bool """ filters = filters or {} q = dict({'assetstoreId': self.assetstore['_id']}, **filters) cursor = File().find(q) progress.update(total=cursor.count(), current=0) for file in cursor: progress.update(increment=1, message=file['name']) path = self.fullPath(file) if not os.path.isfile(path): yield {'reason': 'missing', 'file': file, 'path': path} elif checkSize and os.path.getsize(path) != file['size']: yield {'reason': 'size', 'file': file, 'path': path}