def importData(self, parentId, parentType, public, copyToHome, dataMap, params): user = self.getCurrentUser() if not parentId or parentType not in ('folder', 'item'): parent = getOrCreateRootFolder(CATALOG_NAME) parentType = 'folder' else: parent = self.model(parentType).load(parentId, user=user, level=AccessType.WRITE, exc=True) progress = True importedData = dict(folder=[], item=[]) with ProgressContext(progress, user=user, title='Registering resources') as ctx: for data in dataMap: if data['repository'] == 'DataONE': importedData['folder'].append( register_DataONE_resource(parent, parentType, ctx, user, data['dataId'], name=data['name'])) elif data['repository'] == 'HTTP': importedData['item'].append( register_http_resource(parent, parentType, ctx, user, data['dataId'], data['name'])) if copyToHome: with ProgressContext(progress, user=user, title='Copying to workspace') as ctx: userDataFolder = path_util.lookUpPath( '/user/%s/Data' % user['login'], user) for folder in importedData['folder']: self.model('folder').copyFolder( folder, creator=user, name=folder['name'], parentType='folder', parent=userDataFolder['document'], description=folder['description'], public=folder['public'], progress=ctx) for item in importedData['item']: self.model('item').copyItem( item, creator=user, name=item['name'], folder=userDataFolder['document'], description=item['description'])
def copyFolder(self, folder, parentType, parentId, name, description, public, progress, params): user = self.getCurrentUser() parentType = parentType or folder['parentCollection'] if parentId: parent = self.model(parentType).load(id=parentId, user=user, level=AccessType.WRITE, exc=True) else: parent = None with ProgressContext(progress, user=self.getCurrentUser(), title='Copying folder %s' % folder['name'], message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=self.model('folder').subtreeCount(folder)) return self.model('folder').copyFolder(folder, creator=user, name=name, parentType=parentType, parent=parent, description=description, public=public, progress=ctx)
def buildWebCode(self, params): progress = self.boolParam('progress', params, default=False) dev = self.boolParam('dev', params, default=False) user = self.getCurrentUser() with ProgressContext(progress, user=user, title='Building web client code') as progress: install.runWebBuild(dev=dev, progress=progress)
def moveFileToAssetstore(self, file, assetstore, progress): user = self.getCurrentUser() title = 'Moving file "%s" to assetstore "%s"' % (file['name'], assetstore['name']) with ProgressContext(progress, user=user, title=title, total=file['size']) as ctx: return Upload().moveFileToAssetstore( file=file, user=user, assetstore=assetstore, progress=ctx)
def importData(self, assetstore, params): self.requireParams(('parentId', 'path'), params) user = self.getCurrentUser() parentType = params.get('parentType', 'folder') if parentType not in ('user', 'collection', 'folder'): raise RestException('Invalid parentType.') parent = self.model(parentType).load(params['parentId'], force=True, exc=True) progress = self.boolParam('progress', params, default=False) client = HdfsClient(host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False) path = params['path'] with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx: try: self._importData(parentType, parent, assetstore, client, path, ctx, user) except FileNotFoundException: raise RestException('File not found: %s.' % path)
def updateFolderAccess(self, folder, params): self.requireParams('access', params) user = self.getCurrentUser() public = self.boolParam('public', params) recurse = self.boolParam('recurse', params, default=False) progress = self.boolParam('progress', params, default=False) and recurse try: access = json.loads(params['access']) except ValueError: raise RestException('The access parameter must be JSON.') with ProgressContext(progress, user=user, title='Updating permissions', message='Calculating progress...') as ctx: if progress: ctx.update(total=self.model('folder').subtreeCount( folder, includeItems=False, user=user, level=AccessType.ADMIN)) return self.model('folder').setAccessList(folder, access, save=True, recurse=recurse, user=user, progress=ctx, setPublic=public)
def importData(self, assetstore, params): self.requireParams(('destinationId', 'destinationType'), params) parentType = params.pop('destinationType') if parentType not in ('folder', 'collection', 'user'): raise RestException( 'The destinationType must be user, folder, or collection.') user = self.getCurrentUser() parent = self.model(parentType).load(params.pop('destinationId'), user=user, level=AccessType.ADMIN, exc=True) progress = self.boolParam('progress', params, default=False) leafFoldersAsItems = self.boolParam('leafFoldersAsItems', params, default=False) with ProgressContext(progress, user=user, title='Importing data') as ctx: return self.model('assetstore').importData( assetstore, parent=parent, parentType=parentType, params=params, progress=ctx, user=user, leafFoldersAsItems=leafFoldersAsItems)
def register_dataMap(dataMaps, parent, parentType, user=None, base_url=None): """ Register a list of Data Maps into a given Girder object :param dataMaps: list of dataMaps :param parent: A Collection or a Folder where data should be registered :param parentType: Either a 'collection' or a 'folder' :param user: User performing the registration :param base_url: DataONE's node endpoint url :return: List of ids of registered objects """ progress = True importedData = [] with ProgressContext(progress, user=user, title="Registering resources") as ctx: for dataMap in DataMap.fromList(dataMaps): # probably would be nicer if Entity kept all details and the dataMap # would be merged into it provider = IMPORT_PROVIDERS.getFromDataMap(dataMap) objType, obj = provider.register(parent, parentType, ctx, user, dataMap, base_url=base_url) importedData.append(obj["_id"]) return importedData
def moveResources(self, resources, parentType, parentId, progress): user = self.getCurrentUser() parent = self._prepareMoveOrCopy(resources, parentType, parentId) total = sum([len(resources[key]) for key in resources]) with ProgressContext(progress, user=user, title='Moving resources', message='Calculating requirements...', total=total) as ctx: for kind in resources: model = self._getResourceModel(kind, 'move') for id in resources[kind]: doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True) ctx.update(message='Moving %s %s' % (kind, doc.get('name', ''))) if kind == 'item': if parent['_id'] != doc['folderId']: model.move(doc, parent) elif kind == 'folder': if ((parentType, parent['_id']) != (doc['parentCollection'], doc['parentId'])): model.move(doc, parent, parentType) ctx.update(increment=1)
def copyResources(self, resources, parentType, parentId, progress): user = self.getCurrentUser() parent = self._prepareMoveOrCopy(resources, parentType, parentId) total = len(resources.get('item', [])) if 'folder' in resources: model = self._getResourceModel('folder') for id in resources['folder']: folder = model.load(id=id, user=user, level=AccessType.READ, exc=True) total += model.subtreeCount(folder) with ProgressContext(progress, user=user, title='Copying resources', message='Calculating requirements...', total=total) as ctx: for kind in resources: model = self._getResourceModel(kind) for id in resources[kind]: doc = model.load(id=id, user=user, level=AccessType.READ, exc=True) ctx.update(message='Copying %s %s' % (kind, doc.get('name', ''))) if kind == 'item': model.copyItem(doc, folder=parent, creator=user) ctx.update(increment=1) elif kind == 'folder': model.copyFolder(doc, parent=parent, parentType=parentType, creator=user, progress=ctx)
def copyFolder(self, folder, params): """ Copy an existing folder to a new folder. """ user = self.getCurrentUser() parentType = params.get('parentType', folder['parentCollection']) if 'parentId' in params: parentId = params.get('parentId', folder['parentId']) parent = self.model(parentType).load( id=parentId, user=user, level=AccessType.WRITE, exc=True) else: parent = None name = params.get('name', None) description = params.get('description', None) public = params.get('public', None) progress = self.boolParam('progress', params, default=False) with ProgressContext(progress, user=self.getCurrentUser(), title=u'Copying folder {}'.format(folder['name']), message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=self.model('folder').subtreeCount(folder)) newFolder = self.model('folder').copyFolder( folder, creator=user, name=name, parentType=parentType, parent=parent, description=description, public=public, progress=ctx) return newFolder
def delete(self, resources, progress): user = self.getCurrentUser() self._validateResourceSet(resources, allowedDeleteTypes) total = sum([len(resources[key]) for key in resources]) with ProgressContext(progress, user=user, title='Deleting resources', message='Calculating size...') as ctx: ctx.update(total=total) current = 0 for kind in resources: model = self._getResourceModel(kind, 'remove') for id in resources[kind]: doc = model.load(id=id, user=user, level=AccessType.ADMIN, exc=True) # Don't do a subtree count if we weren't asked for progress if progress: subtotal = model.subtreeCount(doc) if subtotal != 1: total += subtotal - 1 ctx.update(total=total) model.remove(doc, progress=ctx) if progress: current += subtotal if ctx.progress['data']['current'] != current: ctx.update(current=current, message='Deleted ' + kind)
def _handleZip(self, dataset, batch, prereviewFolder, user, zipFile): # Avoid circular import from .image import Image zipFileLocalPath = File().getLocalFilePath(zipFile) with ZipFileOpener(zipFileLocalPath) as (fileList, fileCount): with ProgressContext( on=True, user=user, title='Processing "%s"' % zipFile['name'], total=fileCount, state=ProgressState.ACTIVE, current=0) as progress: for originalFilePath, originalFileRelpath in fileList: originalFileName = os.path.basename(originalFileRelpath) progress.update( increment=1, message='Extracting "%s"' % originalFileName) with open(originalFilePath, 'rb') as originalFileStream: Image().createImage( imageDataStream=originalFileStream, imageDataSize=os.path.getsize(originalFilePath), originalName=originalFileName, parentFolder=prereviewFolder, creator=user, dataset=dataset, batch=batch )
def buildWebCode(self, progress, dev): user = self.getCurrentUser() with ProgressContext(progress, user=user, title='Building web client code') as progress: install.runWebBuild(dev=dev, progress=progress)
def moveResources(self, params): """ Move the specified resources to a new parent folder, user, or collection. Only folder and item resources can be moved with this function. """ user, resources, parent, parentType, progress = \ self._prepareMoveOrCopy(params) total = sum([len(resources[key]) for key in resources]) with ProgressContext(progress, user=user, title='Moving resources', message='Calculating requirements...', total=total) as ctx: for kind in resources: model = self._getResourceModel(kind, 'move') for id in resources[kind]: doc = model.load(id=id, user=user, level=AccessType.WRITE) if not doc: raise RestException('Resource %s %s not found.' % (kind, id)) ctx.update(message='Moving %s %s' % ( kind, doc.get('name', ''))) if kind == 'item': if parent['_id'] != doc['folderId']: model.move(doc, parent) elif kind == 'folder': if ((parentType, parent['_id']) != (doc['parentCollection'], doc['parentId'])): model.move(doc, parent, parentType) ctx.update(increment=1)
def _handleZip(self, prereviewFolder, user, zipFile): # Avoid circular import from .image import Image # Get full path of zip file in assetstore assetstore = Assetstore().getCurrent() assetstore_adapter = assetstore_utilities.getAssetstoreAdapter( assetstore) fullPath = assetstore_adapter.fullPath(zipFile) with ZipFileOpener(fullPath) as (fileList, fileCount): with ProgressContext(on=True, user=user, title='Processing "%s"' % zipFile['name'], total=fileCount, state=ProgressState.ACTIVE, current=0) as progress: for originalFilePath, originalFileRelpath in fileList: originalFileName = os.path.basename(originalFileRelpath) progress.update(increment=1, message='Extracting "%s"' % originalFileName) with open(originalFilePath, 'rb') as originalFileStream: Image().createImage( imageDataStream=originalFileStream, imageDataSize=os.path.getsize(originalFilePath), originalName=originalFileName, parentFolder=prereviewFolder, creator=user)
def copyResources(self, params): """ Copy the specified resources to a new parent folder, user, or collection. Only folder and item resources can be copied with this function. """ user, resources, parent, parentType, progress = \ self._prepareMoveOrCopy(params) total = len(resources.get('item', [])) if 'folder' in resources: model = self._getResourceModel('folder') for id in resources['folder']: folder = model.load(id=id, user=user, level=AccessType.READ) if folder: total += model.subtreeCount(folder) with ProgressContext(progress, user=user, title='Copying resources', message='Calculating requirements...', total=total) as ctx: for kind in resources: model = self._getResourceModel(kind) for id in resources[kind]: doc = model.load(id=id, user=user, level=AccessType.READ) if not doc: raise RestException('Resource not found. No %s with ' 'id %s' % (kind, id)) ctx.update(message='Copying %s %s' % ( kind, doc.get('name', ''))) if kind == 'item': model.copyItem(doc, folder=parent, creator=user) ctx.update(increment=1) elif kind == 'folder': model.copyFolder( doc, parent=parent, parentType=parentType, creator=user, progress=ctx)
def testFilesystemAssetstoreFindInvalidFiles(self): # Create several files in the assetstore, some of which point to real # files on disk and some that don't folder = six.next(Folder().childFolders(parent=self.admin, parentType='user', force=True, limit=1)) item = Item().createItem('test', self.admin, folder) path = os.path.join(ROOT_DIR, 'tests', 'cases', 'py_client', 'testdata', 'hello.txt') real = File().createFile(name='hello.txt', creator=self.admin, item=item, assetstore=self.assetstore, size=os.path.getsize(path)) real['imported'] = True real['path'] = path File().save(real) fake = File().createFile(name='fake', creator=self.admin, item=item, size=1, assetstore=self.assetstore) fake['path'] = 'nonexistent/path/to/file' fake['sha512'] = '...' fake = File().save(fake) fakeImport = File().createFile(name='fakeImport', creator=self.admin, item=item, size=1, assetstore=self.assetstore) fakeImport['imported'] = True fakeImport['path'] = '/nonexistent/path/to/file' fakeImport = File().save(fakeImport) adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore) self.assertTrue(inspect.isgeneratorfunction(adapter.findInvalidFiles)) with ProgressContext(True, user=self.admin, title='test') as p: invalidFiles = list( adapter.findInvalidFiles(progress=p, filters={'imported': True})) self.assertEqual(len(invalidFiles), 1) self.assertEqual(invalidFiles[0]['reason'], 'missing') self.assertEqual(invalidFiles[0]['file']['_id'], fakeImport['_id']) self.assertEqual(p.progress['data']['current'], 2) self.assertEqual(p.progress['data']['total'], 2) invalidFiles = list(adapter.findInvalidFiles(progress=p)) self.assertEqual(len(invalidFiles), 2) for invalidFile in invalidFiles: self.assertEqual(invalidFile['reason'], 'missing') self.assertIn(invalidFile['file']['_id'], (fakeImport['_id'], fake['_id'])) self.assertEqual(p.progress['data']['current'], 3) self.assertEqual(p.progress['data']['total'], 3)
def deleteContents(self, folder, progress): with ProgressContext(progress, user=self.getCurrentUser(), title='Clearing folder %s' % folder['name'], message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=self._model.subtreeCount(folder) - 1) self._model.clean(folder, progress=ctx) return {'message': 'Cleaned folder %s.' % folder['name']}
def deleteFolder(self, folder, progress, params): with ProgressContext(progress, user=self.getCurrentUser(), title='Deleting folder %s' % folder['name'], message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=self.model('folder').subtreeCount(folder)) self.model('folder').remove(folder, progress=ctx) return {'message': 'Deleted folder %s.' % folder['name']}
def deleteFolder(self, folder, params): progress = self.boolParam('progress', params, default=False) with ProgressContext(progress, user=self.getCurrentUser(), title=u'Deleting folder {}'.format(folder['name']), message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=self.model('folder').subtreeCount(folder)) self.model('folder').remove(folder, progress=ctx) return {'message': u'Deleted folder {}.'.format(folder['name'])}
def _deleteFolder(folder, progress, user): with ProgressContext(progress, user=user, title='Deleting folder %s' % folder['name'], message='Calculating folder size...') as ctx: # Don't do the subtree count if we weren't asked for progress if progress: ctx.update(total=Folder().subtreeCount(folder)) Folder().remove(folder, progress=ctx) return {'message': 'Deleted folder %s.' % folder['name']}
def updateFolderAccess(self, folder, access, publicFlags, public, recurse, progress): user = self.getCurrentUser() progress = progress and recurse # Only enable progress in recursive case with ProgressContext(progress, user=user, title='Updating permissions', message='Calculating progress...') as ctx: if progress: ctx.update(total=self._model.subtreeCount( folder, includeItems=False, user=user, level=AccessType.ADMIN)) return self._model.setAccessList( folder, access, save=True, recurse=recurse, user=user, progress=ctx, setPublic=public, publicFlags=publicFlags)
def deletePhase(self, phase, params): progress = self.boolParam('progress', params, default=False) with ProgressContext(progress, user=self.getCurrentUser(), title=u'Deleting phase ' + phase['name'], message='Calculating total size...') as ctx: if progress: ctx.update( total=self.model('phase', 'covalic').subtreeCount(phase)) self.model('phase', 'covalic').remove(phase, progress=ctx) return {'message': 'Deleted phase %s.' % phase['name']}
def systemConsistencyCheck(self, progress): user = self.getCurrentUser() title = 'Running system consistency check' with ProgressContext(progress, user=user, title=title) as pc: results = {} pc.update(title='Checking for orphaned records (Step 1 of 3)') results['orphansRemoved'] = self._pruneOrphans(pc) pc.update(title='Checking for incorrect base parents (Step 2 of 3)') results['baseParentsFixed'] = self._fixBaseParents(pc) pc.update(title='Checking for incorrect sizes (Step 3 of 3)') results['sizesChanged'] = self._recalculateSizes(pc) return results
def updateCollectionAccess( self, collection, access, public, recurse, progress, publicFlags): user = self.getCurrentUser() progress = progress and recurse with ProgressContext(progress, user=user, title='Updating permissions', message='Calculating progress...') as ctx: if progress: ctx.update(total=self.model('collection').subtreeCount( collection, includeItems=False, user=user, level=AccessType.ADMIN)) return self.model('collection').setAccessList( collection, access, save=True, user=user, recurse=recurse, progress=ctx, setPublic=public, publicFlags=publicFlags)
def deleteTale(self, tale, progress): user = self.getCurrentUser() workspace = Folder().load(tale['workspaceId'], user=user, level=AccessType.ADMIN) with ProgressContext( progress, user=user, title='Deleting workspace of {title}'.format(**tale), message='Calculating folder size...') as ctx: if progress: ctx.update(total=Folder().subtreeCount(workspace)) Folder().remove(workspace, progress=ctx) self._model.remove(tale)
def migrate_item(item): _file = list(Item().childFiles(item))[0] url = _file['linkUrl'] if url.startswith('https://dashboard.wholetale.org'): return 0 creator = User().load(item['creatorId'], force=True) # register url entity = Entity(url.strip(), creator) provider = HTTPImportProvider() try: dataMap = provider.lookup(entity) except Exception: print(" -> Failed to resolve {} as HTTP".format(url)) print(" -> item_id = {}".format(str(item["_id"]))) return 0 ds = dataMap.toDict() if not ds['name']: print(" -> Item has no name!!!") print(ds) return 0 with ProgressContext(True, user=creator, title='Registering resources') as ctx: objType, new_item = provider.register(CAT_ROOT, 'folder', ctx, creator, dataMap, base_url=base_url) # find userData and replace with new id for user in User().find({'myData': item['_id']}): print(' Updating {} in myData for user "{}"'.format( item['name'], user['login'])) user['myData'][user['myData'].index(item['_id'])] = new_item['_id'] user = User().save(user) # find tale dataset and switch id for tale in Tale().find({'dataSet.itemId': str(item['_id'])}): print(' Updating {} in dataSet of Tale: "{}"'.format( item['name'], tale['title'])) for i, ds in enumerate(tale['dataSet']): if ds['itemId'] == str(item['_id']): tale['dataSet'][i]['itemId'] = str(new_item['_id']) Tale().save(tale) return 1
def testProgress(self, params): test = params.get('test', 'success') duration = int(params.get('duration', 10)) startTime = time.time() with ProgressContext(True, user=self.getCurrentUser(), title='Progress Test', message='Progress Message', total=duration) as ctx: for current in range(duration): if self.stop: break ctx.update(current=current) wait = startTime + current + 1 - time.time() if wait > 0: time.sleep(wait) if test == 'error': raise RestException('Progress error test.')
def move_files_to_assetstore(files, totalsize): user, token = getCurrentUser(True) assetstore = ModelImporter.model('assetstore').getCurrent() upload = ModelImporter.model('upload') with ProgressContext(True, interval=1.0, message=files[0]['name'], user=user, token=token, total=totalsize, title='Moving files') as ctx: for file in files: ctx.update(message=file['name']) upload.moveFileToAssetstore(file=file, user=user, assetstore=assetstore, progress=ctx)