def importData(self, parentId, parentType, public, copyToHome, dataMap,
params):
user = self.getCurrentUser()
if not parentId or parentType not in ('folder', 'item'):
parent = getOrCreateRootFolder(CATALOG_NAME)
parentType = 'folder'
else:
parent = self.model(parentType).load(parentId,
user=user,
level=AccessType.WRITE,
exc=True)
progress = True
importedData = dict(folder=[], item=[])
with ProgressContext(progress,
user=user,
title='Registering resources') as ctx:
for data in dataMap:
if data['repository'] == 'DataONE':
importedData['folder'].append(
register_DataONE_resource(parent,
parentType,
ctx,
user,
data['dataId'],
name=data['name']))
elif data['repository'] == 'HTTP':
importedData['item'].append(
register_http_resource(parent, parentType, ctx, user,
data['dataId'], data['name']))
if copyToHome:
with ProgressContext(progress,
user=user,
title='Copying to workspace') as ctx:
userDataFolder = path_util.lookUpPath(
'/user/%s/Data' % user['login'], user)
for folder in importedData['folder']:
self.model('folder').copyFolder(
folder,
creator=user,
name=folder['name'],
parentType='folder',
parent=userDataFolder['document'],
description=folder['description'],
public=folder['public'],
progress=ctx)
for item in importedData['item']:
self.model('item').copyItem(
item,
creator=user,
name=item['name'],
folder=userDataFolder['document'],
description=item['description'])
def copyFolder(self, folder, parentType, parentId, name, description,
public, progress, params):
user = self.getCurrentUser()
parentType = parentType or folder['parentCollection']
if parentId:
parent = self.model(parentType).load(id=parentId,
user=user,
level=AccessType.WRITE,
exc=True)
else:
parent = None
with ProgressContext(progress,
user=self.getCurrentUser(),
title='Copying folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self.model('folder').subtreeCount(folder))
return self.model('folder').copyFolder(folder,
creator=user,
name=name,
parentType=parentType,
parent=parent,
description=description,
public=public,
progress=ctx)
def buildWebCode(self, params):
progress = self.boolParam('progress', params, default=False)
dev = self.boolParam('dev', params, default=False)
user = self.getCurrentUser()
with ProgressContext(progress, user=user, title='Building web client code') as progress:
install.runWebBuild(dev=dev, progress=progress)
def moveFileToAssetstore(self, file, assetstore, progress):
user = self.getCurrentUser()
title = 'Moving file "%s" to assetstore "%s"' % (file['name'], assetstore['name'])
with ProgressContext(progress, user=user, title=title, total=file['size']) as ctx:
return Upload().moveFileToAssetstore(
file=file, user=user, assetstore=assetstore, progress=ctx)
def importData(self, assetstore, params):
self.requireParams(('parentId', 'path'), params)
user = self.getCurrentUser()
parentType = params.get('parentType', 'folder')
if parentType not in ('user', 'collection', 'folder'):
raise RestException('Invalid parentType.')
parent = self.model(parentType).load(params['parentId'],
force=True,
exc=True)
progress = self.boolParam('progress', params, default=False)
client = HdfsClient(host=assetstore['hdfs']['host'],
port=assetstore['hdfs']['port'],
use_trash=False)
path = params['path']
with ProgressContext(progress,
user=user,
title='Importing data from HDFS') as ctx:
try:
self._importData(parentType, parent, assetstore, client, path,
ctx, user)
except FileNotFoundException:
raise RestException('File not found: %s.' % path)
def updateFolderAccess(self, folder, params):
self.requireParams('access', params)
user = self.getCurrentUser()
public = self.boolParam('public', params)
recurse = self.boolParam('recurse', params, default=False)
progress = self.boolParam('progress', params,
default=False) and recurse
try:
access = json.loads(params['access'])
except ValueError:
raise RestException('The access parameter must be JSON.')
with ProgressContext(progress,
user=user,
title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self.model('folder').subtreeCount(
folder,
includeItems=False,
user=user,
level=AccessType.ADMIN))
return self.model('folder').setAccessList(folder,
access,
save=True,
recurse=recurse,
user=user,
progress=ctx,
setPublic=public)
def importData(self, assetstore, params):
self.requireParams(('destinationId', 'destinationType'), params)
parentType = params.pop('destinationType')
if parentType not in ('folder', 'collection', 'user'):
raise RestException(
'The destinationType must be user, folder, or collection.')
user = self.getCurrentUser()
parent = self.model(parentType).load(params.pop('destinationId'),
user=user,
level=AccessType.ADMIN,
exc=True)
progress = self.boolParam('progress', params, default=False)
leafFoldersAsItems = self.boolParam('leafFoldersAsItems',
params,
default=False)
with ProgressContext(progress, user=user,
title='Importing data') as ctx:
return self.model('assetstore').importData(
assetstore,
parent=parent,
parentType=parentType,
params=params,
progress=ctx,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
def register_dataMap(dataMaps, parent, parentType, user=None, base_url=None):
"""
Register a list of Data Maps into a given Girder object
:param dataMaps: list of dataMaps
:param parent: A Collection or a Folder where data should be registered
:param parentType: Either a 'collection' or a 'folder'
:param user: User performing the registration
:param base_url: DataONE's node endpoint url
:return: List of ids of registered objects
"""
progress = True
importedData = []
with ProgressContext(progress, user=user,
title="Registering resources") as ctx:
for dataMap in DataMap.fromList(dataMaps):
# probably would be nicer if Entity kept all details and the dataMap
# would be merged into it
provider = IMPORT_PROVIDERS.getFromDataMap(dataMap)
objType, obj = provider.register(parent,
parentType,
ctx,
user,
dataMap,
base_url=base_url)
importedData.append(obj["_id"])
return importedData
def moveResources(self, resources, parentType, parentId, progress):
user = self.getCurrentUser()
parent = self._prepareMoveOrCopy(resources, parentType, parentId)
total = sum([len(resources[key]) for key in resources])
with ProgressContext(progress,
user=user,
title='Moving resources',
message='Calculating requirements...',
total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind, 'move')
for id in resources[kind]:
doc = model.load(id=id,
user=user,
level=AccessType.WRITE,
exc=True)
ctx.update(message='Moving %s %s' %
(kind, doc.get('name', '')))
if kind == 'item':
if parent['_id'] != doc['folderId']:
model.move(doc, parent)
elif kind == 'folder':
if ((parentType, parent['_id']) !=
(doc['parentCollection'], doc['parentId'])):
model.move(doc, parent, parentType)
ctx.update(increment=1)
def copyResources(self, resources, parentType, parentId, progress):
user = self.getCurrentUser()
parent = self._prepareMoveOrCopy(resources, parentType, parentId)
total = len(resources.get('item', []))
if 'folder' in resources:
model = self._getResourceModel('folder')
for id in resources['folder']:
folder = model.load(id=id,
user=user,
level=AccessType.READ,
exc=True)
total += model.subtreeCount(folder)
with ProgressContext(progress,
user=user,
title='Copying resources',
message='Calculating requirements...',
total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind)
for id in resources[kind]:
doc = model.load(id=id,
user=user,
level=AccessType.READ,
exc=True)
ctx.update(message='Copying %s %s' %
(kind, doc.get('name', '')))
if kind == 'item':
model.copyItem(doc, folder=parent, creator=user)
ctx.update(increment=1)
elif kind == 'folder':
model.copyFolder(doc,
parent=parent,
parentType=parentType,
creator=user,
progress=ctx)
def copyFolder(self, folder, params):
"""
Copy an existing folder to a new folder.
"""
user = self.getCurrentUser()
parentType = params.get('parentType', folder['parentCollection'])
if 'parentId' in params:
parentId = params.get('parentId', folder['parentId'])
parent = self.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
else:
parent = None
name = params.get('name', None)
description = params.get('description', None)
public = params.get('public', None)
progress = self.boolParam('progress', params, default=False)
with ProgressContext(progress, user=self.getCurrentUser(),
title=u'Copying folder {}'.format(folder['name']),
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self.model('folder').subtreeCount(folder))
newFolder = self.model('folder').copyFolder(
folder, creator=user, name=name, parentType=parentType,
parent=parent, description=description, public=public,
progress=ctx)
return newFolder
def delete(self, resources, progress):
user = self.getCurrentUser()
self._validateResourceSet(resources, allowedDeleteTypes)
total = sum([len(resources[key]) for key in resources])
with ProgressContext(progress,
user=user,
title='Deleting resources',
message='Calculating size...') as ctx:
ctx.update(total=total)
current = 0
for kind in resources:
model = self._getResourceModel(kind, 'remove')
for id in resources[kind]:
doc = model.load(id=id,
user=user,
level=AccessType.ADMIN,
exc=True)
# Don't do a subtree count if we weren't asked for progress
if progress:
subtotal = model.subtreeCount(doc)
if subtotal != 1:
total += subtotal - 1
ctx.update(total=total)
model.remove(doc, progress=ctx)
if progress:
current += subtotal
if ctx.progress['data']['current'] != current:
ctx.update(current=current,
message='Deleted ' + kind)
def _handleZip(self, dataset, batch, prereviewFolder, user, zipFile):
# Avoid circular import
from .image import Image
zipFileLocalPath = File().getLocalFilePath(zipFile)
with ZipFileOpener(zipFileLocalPath) as (fileList, fileCount):
with ProgressContext(
on=True,
user=user,
title='Processing "%s"' % zipFile['name'],
total=fileCount,
state=ProgressState.ACTIVE,
current=0) as progress:
for originalFilePath, originalFileRelpath in fileList:
originalFileName = os.path.basename(originalFileRelpath)
progress.update(
increment=1,
message='Extracting "%s"' % originalFileName)
with open(originalFilePath, 'rb') as originalFileStream:
Image().createImage(
imageDataStream=originalFileStream,
imageDataSize=os.path.getsize(originalFilePath),
originalName=originalFileName,
parentFolder=prereviewFolder,
creator=user,
dataset=dataset,
batch=batch
)
def buildWebCode(self, progress, dev):
user = self.getCurrentUser()
with ProgressContext(progress,
user=user,
title='Building web client code') as progress:
install.runWebBuild(dev=dev, progress=progress)
def moveResources(self, params):
"""
Move the specified resources to a new parent folder, user, or
collection. Only folder and item resources can be moved with this
function.
"""
user, resources, parent, parentType, progress = \
self._prepareMoveOrCopy(params)
total = sum([len(resources[key]) for key in resources])
with ProgressContext(progress, user=user, title='Moving resources',
message='Calculating requirements...',
total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind, 'move')
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.WRITE)
if not doc:
raise RestException('Resource %s %s not found.' %
(kind, id))
ctx.update(message='Moving %s %s' % (
kind, doc.get('name', '')))
if kind == 'item':
if parent['_id'] != doc['folderId']:
model.move(doc, parent)
elif kind == 'folder':
if ((parentType, parent['_id']) !=
(doc['parentCollection'], doc['parentId'])):
model.move(doc, parent, parentType)
ctx.update(increment=1)
def _handleZip(self, prereviewFolder, user, zipFile):
# Avoid circular import
from .image import Image
# Get full path of zip file in assetstore
assetstore = Assetstore().getCurrent()
assetstore_adapter = assetstore_utilities.getAssetstoreAdapter(
assetstore)
fullPath = assetstore_adapter.fullPath(zipFile)
with ZipFileOpener(fullPath) as (fileList, fileCount):
with ProgressContext(on=True,
user=user,
title='Processing "%s"' % zipFile['name'],
total=fileCount,
state=ProgressState.ACTIVE,
current=0) as progress:
for originalFilePath, originalFileRelpath in fileList:
originalFileName = os.path.basename(originalFileRelpath)
progress.update(increment=1,
message='Extracting "%s"' %
originalFileName)
with open(originalFilePath, 'rb') as originalFileStream:
Image().createImage(
imageDataStream=originalFileStream,
imageDataSize=os.path.getsize(originalFilePath),
originalName=originalFileName,
parentFolder=prereviewFolder,
creator=user)
def copyResources(self, params):
"""
Copy the specified resources to a new parent folder, user, or
collection. Only folder and item resources can be copied with this
function.
"""
user, resources, parent, parentType, progress = \
self._prepareMoveOrCopy(params)
total = len(resources.get('item', []))
if 'folder' in resources:
model = self._getResourceModel('folder')
for id in resources['folder']:
folder = model.load(id=id, user=user,
level=AccessType.READ)
if folder:
total += model.subtreeCount(folder)
with ProgressContext(progress, user=user, title='Copying resources',
message='Calculating requirements...',
total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind)
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.READ)
if not doc:
raise RestException('Resource not found. No %s with '
'id %s' % (kind, id))
ctx.update(message='Copying %s %s' % (
kind, doc.get('name', '')))
if kind == 'item':
model.copyItem(doc, folder=parent, creator=user)
ctx.update(increment=1)
elif kind == 'folder':
model.copyFolder(
doc, parent=parent, parentType=parentType,
creator=user, progress=ctx)
def testFilesystemAssetstoreFindInvalidFiles(self):
# Create several files in the assetstore, some of which point to real
# files on disk and some that don't
folder = six.next(Folder().childFolders(parent=self.admin,
parentType='user',
force=True,
limit=1))
item = Item().createItem('test', self.admin, folder)
path = os.path.join(ROOT_DIR, 'tests', 'cases', 'py_client',
'testdata', 'hello.txt')
real = File().createFile(name='hello.txt',
creator=self.admin,
item=item,
assetstore=self.assetstore,
size=os.path.getsize(path))
real['imported'] = True
real['path'] = path
File().save(real)
fake = File().createFile(name='fake',
creator=self.admin,
item=item,
size=1,
assetstore=self.assetstore)
fake['path'] = 'nonexistent/path/to/file'
fake['sha512'] = '...'
fake = File().save(fake)
fakeImport = File().createFile(name='fakeImport',
creator=self.admin,
item=item,
size=1,
assetstore=self.assetstore)
fakeImport['imported'] = True
fakeImport['path'] = '/nonexistent/path/to/file'
fakeImport = File().save(fakeImport)
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
self.assertTrue(inspect.isgeneratorfunction(adapter.findInvalidFiles))
with ProgressContext(True, user=self.admin, title='test') as p:
invalidFiles = list(
adapter.findInvalidFiles(progress=p,
filters={'imported': True}))
self.assertEqual(len(invalidFiles), 1)
self.assertEqual(invalidFiles[0]['reason'], 'missing')
self.assertEqual(invalidFiles[0]['file']['_id'], fakeImport['_id'])
self.assertEqual(p.progress['data']['current'], 2)
self.assertEqual(p.progress['data']['total'], 2)
invalidFiles = list(adapter.findInvalidFiles(progress=p))
self.assertEqual(len(invalidFiles), 2)
for invalidFile in invalidFiles:
self.assertEqual(invalidFile['reason'], 'missing')
self.assertIn(invalidFile['file']['_id'],
(fakeImport['_id'], fake['_id']))
self.assertEqual(p.progress['data']['current'], 3)
self.assertEqual(p.progress['data']['total'], 3)
def deleteContents(self, folder, progress):
with ProgressContext(progress, user=self.getCurrentUser(),
title='Clearing folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder) - 1)
self._model.clean(folder, progress=ctx)
return {'message': 'Cleaned folder %s.' % folder['name']}
def deleteFolder(self, folder, progress, params):
with ProgressContext(progress, user=self.getCurrentUser(),
title='Deleting folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self.model('folder').subtreeCount(folder))
self.model('folder').remove(folder, progress=ctx)
return {'message': 'Deleted folder %s.' % folder['name']}
def deleteFolder(self, folder, params):
progress = self.boolParam('progress', params, default=False)
with ProgressContext(progress, user=self.getCurrentUser(),
title=u'Deleting folder {}'.format(folder['name']),
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self.model('folder').subtreeCount(folder))
self.model('folder').remove(folder, progress=ctx)
return {'message': u'Deleted folder {}.'.format(folder['name'])}
def _deleteFolder(folder, progress, user):
with ProgressContext(progress,
user=user,
title='Deleting folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=Folder().subtreeCount(folder))
Folder().remove(folder, progress=ctx)
return {'message': 'Deleted folder %s.' % folder['name']}
def updateFolderAccess(self, folder, access, publicFlags, public, recurse, progress):
user = self.getCurrentUser()
progress = progress and recurse # Only enable progress in recursive case
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self._model.subtreeCount(
folder, includeItems=False, user=user, level=AccessType.ADMIN))
return self._model.setAccessList(
folder, access, save=True, recurse=recurse, user=user,
progress=ctx, setPublic=public, publicFlags=publicFlags)
def deletePhase(self, phase, params):
progress = self.boolParam('progress', params, default=False)
with ProgressContext(progress,
user=self.getCurrentUser(),
title=u'Deleting phase ' + phase['name'],
message='Calculating total size...') as ctx:
if progress:
ctx.update(
total=self.model('phase', 'covalic').subtreeCount(phase))
self.model('phase', 'covalic').remove(phase, progress=ctx)
return {'message': 'Deleted phase %s.' % phase['name']}
def systemConsistencyCheck(self, progress):
user = self.getCurrentUser()
title = 'Running system consistency check'
with ProgressContext(progress, user=user, title=title) as pc:
results = {}
pc.update(title='Checking for orphaned records (Step 1 of 3)')
results['orphansRemoved'] = self._pruneOrphans(pc)
pc.update(title='Checking for incorrect base parents (Step 2 of 3)')
results['baseParentsFixed'] = self._fixBaseParents(pc)
pc.update(title='Checking for incorrect sizes (Step 3 of 3)')
results['sizesChanged'] = self._recalculateSizes(pc)
return results
def updateCollectionAccess(
self, collection, access, public, recurse, progress, publicFlags):
user = self.getCurrentUser()
progress = progress and recurse
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self.model('collection').subtreeCount(
collection, includeItems=False, user=user,
level=AccessType.ADMIN))
return self.model('collection').setAccessList(
collection, access, save=True, user=user, recurse=recurse,
progress=ctx, setPublic=public, publicFlags=publicFlags)
def deleteTale(self, tale, progress):
user = self.getCurrentUser()
workspace = Folder().load(tale['workspaceId'],
user=user,
level=AccessType.ADMIN)
with ProgressContext(
progress,
user=user,
title='Deleting workspace of {title}'.format(**tale),
message='Calculating folder size...') as ctx:
if progress:
ctx.update(total=Folder().subtreeCount(workspace))
Folder().remove(workspace, progress=ctx)
self._model.remove(tale)
def migrate_item(item):
_file = list(Item().childFiles(item))[0]
url = _file['linkUrl']
if url.startswith('https://dashboard.wholetale.org'):
return 0
creator = User().load(item['creatorId'], force=True)
# register url
entity = Entity(url.strip(), creator)
provider = HTTPImportProvider()
try:
dataMap = provider.lookup(entity)
except Exception:
print(" -> Failed to resolve {} as HTTP".format(url))
print(" -> item_id = {}".format(str(item["_id"])))
return 0
ds = dataMap.toDict()
if not ds['name']:
print(" -> Item has no name!!!")
print(ds)
return 0
with ProgressContext(True, user=creator,
title='Registering resources') as ctx:
objType, new_item = provider.register(CAT_ROOT,
'folder',
ctx,
creator,
dataMap,
base_url=base_url)
# find userData and replace with new id
for user in User().find({'myData': item['_id']}):
print(' Updating {} in myData for user "{}"'.format(
item['name'], user['login']))
user['myData'][user['myData'].index(item['_id'])] = new_item['_id']
user = User().save(user)
# find tale dataset and switch id
for tale in Tale().find({'dataSet.itemId': str(item['_id'])}):
print(' Updating {} in dataSet of Tale: "{}"'.format(
item['name'], tale['title']))
for i, ds in enumerate(tale['dataSet']):
if ds['itemId'] == str(item['_id']):
tale['dataSet'][i]['itemId'] = str(new_item['_id'])
Tale().save(tale)
return 1
def testProgress(self, params):
test = params.get('test', 'success')
duration = int(params.get('duration', 10))
startTime = time.time()
with ProgressContext(True, user=self.getCurrentUser(),
title='Progress Test', message='Progress Message',
total=duration) as ctx:
for current in range(duration):
if self.stop:
break
ctx.update(current=current)
wait = startTime + current + 1 - time.time()
if wait > 0:
time.sleep(wait)
if test == 'error':
raise RestException('Progress error test.')
def move_files_to_assetstore(files, totalsize):
user, token = getCurrentUser(True)
assetstore = ModelImporter.model('assetstore').getCurrent()
upload = ModelImporter.model('upload')
with ProgressContext(True,
interval=1.0,
message=files[0]['name'],
user=user,
token=token,
total=totalsize,
title='Moving files') as ctx:
for file in files:
ctx.update(message=file['name'])
upload.moveFileToAssetstore(file=file,
user=user,
assetstore=assetstore,
progress=ctx)
