def updateFolderAccess(self, folder, access, publicFlags, public, recurse, progress, params):
user = self.getCurrentUser()
progress = progress and recurse # Only enable progress in recursive case
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self.model('folder').subtreeCount(
folder, includeItems=False, user=user,
level=AccessType.ADMIN))
return self.model('folder').setAccessList(
folder, access, save=True, recurse=recurse, user=user,
progress=ctx, setPublic=public, publicFlags=publicFlags)
def updateCollectionAccess(self, collection, access, public, recurse, progress, publicFlags):
user = self.getCurrentUser()
progress = progress and recurse
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self._model.subtreeCount(
collection, includeItems=False, user=user, level=AccessType.ADMIN))
return self._model.setAccessList(
collection, access, save=True, user=user, recurse=recurse,
progress=ctx, setPublic=public, publicFlags=publicFlags)
def systemConsistencyCheck(self, progress):
user = self.getCurrentUser()
title = 'Running system consistency check'
with ProgressContext(progress, user=user, title=title) as pc:
results = {}
pc.update(title='Checking for orphaned records (Step 1 of 3)')
results['orphansRemoved'] = self._pruneOrphans(pc)
pc.update(title='Checking for incorrect base parents (Step 2 of 3)')
results['baseParentsFixed'] = self._fixBaseParents(pc)
pc.update(title='Checking for incorrect sizes (Step 3 of 3)')
results['sizesChanged'] = self._recalculateSizes(pc)
return results
def moveFileToAssetstore(self, file, assetstore, progress):
user = self.getCurrentUser()
title = 'Moving file "%s" to assetstore "%s"' % (file['name'],
assetstore['name'])
with ProgressContext(progress,
user=user,
title=title,
total=file['size']) as ctx:
return Upload().moveFileToAssetstore(file=file,
user=user,
assetstore=assetstore,
progress=ctx)
def deleteTale(self, tale, progress):
user = self.getCurrentUser()
workspace = Folder().load(tale['workspaceId'],
user=user,
level=AccessType.ADMIN)
with ProgressContext(
progress,
user=user,
title='Deleting workspace of {title}'.format(**tale),
message='Calculating folder size...') as ctx:
if progress:
ctx.update(total=Folder().subtreeCount(workspace))
Folder().remove(workspace, progress=ctx)
self._model.remove(tale)
def testFilesystemAssetstoreFindInvalidFiles(self):
# Create several files in the assetstore, some of which point to real
# files on disk and some that don't
folder = six.next(self.model('folder').childFolders(
parent=self.admin, parentType='user', force=True, limit=1))
item = self.model('item').createItem('test', self.admin, folder)
path = os.path.join(
ROOT_DIR, 'tests', 'cases', 'py_client', 'testdata', 'hello.txt')
real = self.model('file').createFile(
name='hello.txt', creator=self.admin, item=item,
assetstore=self.assetstore, size=os.path.getsize(path))
real['imported'] = True
real['path'] = path
self.model('file').save(real)
fake = self.model('file').createFile(
name='fake', creator=self.admin, item=item, size=1, assetstore=self.assetstore)
fake['path'] = 'nonexistent/path/to/file'
fake['sha512'] = '...'
fake = self.model('file').save(fake)
fakeImport = self.model('file').createFile(
name='fakeImport', creator=self.admin, item=item, size=1, assetstore=self.assetstore)
fakeImport['imported'] = True
fakeImport['path'] = '/nonexistent/path/to/file'
fakeImport = self.model('file').save(fakeImport)
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
self.assertTrue(inspect.isgeneratorfunction(adapter.findInvalidFiles))
with ProgressContext(True, user=self.admin, title='test') as p:
for i, info in enumerate(
adapter.findInvalidFiles(progress=p, filters={
'imported': True
}), 1):
self.assertEqual(info['reason'], 'missing')
self.assertEqual(info['file']['_id'], fakeImport['_id'])
self.assertEqual(i, 1)
self.assertEqual(p.progress['data']['current'], 2)
self.assertEqual(p.progress['data']['total'], 2)
for i, info in enumerate(
adapter.findInvalidFiles(progress=p), 1):
self.assertEqual(info['reason'], 'missing')
self.assertIn(info['file']['_id'], (fakeImport['_id'], fake['_id']))
self.assertEqual(i, 2)
self.assertEqual(p.progress['data']['current'], 3)
self.assertEqual(p.progress['data']['total'], 3)
def migrate_item(item):
_file = list(Item().childFiles(item))[0]
url = _file['linkUrl']
if url.startswith('https://dashboard.wholetale.org'):
return 0
creator = User().load(item['creatorId'], force=True)
# register url
entity = Entity(url.strip(), creator)
provider = HTTPImportProvider()
try:
dataMap = provider.lookup(entity)
except Exception:
print(" -> Failed to resolve {} as HTTP".format(url))
print(" -> item_id = {}".format(str(item["_id"])))
return 0
ds = dataMap.toDict()
if not ds['name']:
print(" -> Item has no name!!!")
print(ds)
return 0
with ProgressContext(True, user=creator,
title='Registering resources') as ctx:
objType, new_item = provider.register(CAT_ROOT,
'folder',
ctx,
creator,
dataMap,
base_url=base_url)
# find userData and replace with new id
for user in User().find({'myData': item['_id']}):
print(' Updating {} in myData for user "{}"'.format(
item['name'], user['login']))
user['myData'][user['myData'].index(item['_id'])] = new_item['_id']
user = User().save(user)
# find tale dataset and switch id
for tale in Tale().find({'dataSet.itemId': str(item['_id'])}):
print(' Updating {} in dataSet of Tale: "{}"'.format(
item['name'], tale['title']))
for i, ds in enumerate(tale['dataSet']):
if ds['itemId'] == str(item['_id']):
tale['dataSet'][i]['itemId'] = str(new_item['_id'])
Tale().save(tale)
return 1
def testProgress(self, params):
test = params.get('test', 'success')
duration = int(params.get('duration', 10))
startTime = time.time()
with ProgressContext(True, user=self.getCurrentUser(),
title='Progress Test', message='Progress Message',
total=duration) as ctx:
for current in range(duration):
if self.stop:
break
ctx.update(current=current)
wait = startTime + current + 1 - time.time()
if wait > 0:
time.sleep(wait)
if test == 'error':
raise RestException('Progress error test.')
def move_files_to_assetstore(files, totalsize):
user, token = getCurrentUser(True)
assetstore = ModelImporter.model('assetstore').getCurrent()
upload = ModelImporter.model('upload')
with ProgressContext(True,
interval=1.0,
message=files[0]['name'],
user=user,
token=token,
total=totalsize,
title='Moving files') as ctx:
for file in files:
ctx.update(message=file['name'])
upload.moveFileToAssetstore(file=file,
user=user,
assetstore=assetstore,
progress=ctx)
def move_resources(self, event, path, root, user=None):
wt_resources = self._filter_resources(event, level=AccessType.WRITE, user=user)
progress = event.info["params"].get("progress", False)
with ProgressContext(
progress,
user=user,
title="Moving resources",
message="Calculating requirements...",
total=len(wt_resources),
) as ctx:
for obj in wt_resources:
source_path = obj["src_path"]
ctx.update(message="Moving %s %s" % (obj["kind"], source_path.name))
shutil.move(
source_path.as_posix(), (path / source_path.name).as_posix()
)
ctx.update(increment=1)
def copyFolder(self, folder, parentType, parentId, name, description, public, progress):
user = self.getCurrentUser()
parentType = parentType or folder['parentCollection']
if parentId:
parent = self.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
else:
parent = None
with ProgressContext(progress, user=self.getCurrentUser(),
title='Copying folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self._model.subtreeCount(folder))
return self._model.copyFolder(
folder, creator=user, name=name, parentType=parentType,
parent=parent, description=description, public=public, progress=ctx)
def _importTar(self, assetstore, folder, path, progress):
importGroupId = Setting().get(WHITELIST_GROUP_SETTING)
if not importGroupId:
raise Exception('Import whitelist group ID is not set')
user = self.getCurrentUser()
if importGroupId not in user['groups']:
raise AccessException('You are not authorized to import tape archive files.')
if assetstore is None:
# This is a reasonable fallback behavior, but we may want something more robust.
# Imported files are weird anyway
assetstore = Assetstore().getCurrent()
if assetstore['type'] != AssetstoreType.FILESYSTEM:
raise Exception('Not a filesystem assetstore: %s' % assetstore['_id'])
with ProgressContext(progress, user=user, title='Importing data') as ctx:
getAssetstoreAdapter(assetstore)._importTar(path, folder, ctx, user)
def importData(self, assetstore, params):
self.requireParams(('destinationId', 'destinationType'), params)
parentType = params.pop('destinationType')
if parentType not in ('folder', 'collection', 'user'):
raise RestException('The destinationType must be user, folder, or '
'collection.')
user = self.getCurrentUser()
parent = self.model(parentType).load(
params.pop('destinationId'), user=user, level=AccessType.ADMIN,
exc=True)
progress = self.boolParam('progress', params, default=False)
with ProgressContext(
progress, user=user, title='Importing data') as ctx:
return self.model('assetstore').importData(
assetstore, parent=parent, parentType=parentType, params=params,
progress=ctx, user=user)
def systemConsistencyCheck(self, params):
results = {}
progress = self.boolParam('progress', params, default=False)
models = ('item', )
steps = 0
if progress:
for model in models:
count, changed = self.model(model).checkConsistency(
stage='count')
steps += count
with ProgressContext(progress,
user=self.getCurrentUser(),
title='Checking system',
total=steps,
message='Checking system...') as ctx:
for model in models:
count, removed = self.model(model).checkConsistency(
stage='remove', progress=ctx)
if removed:
results[model + 'Removed'] = removed
revmodels = list(models)
revmodels.reverse()
for model in revmodels:
count, corrected = self.model(model).checkConsistency(
stage='verify', progress=ctx)
if count:
results[model + 'Count'] = count
if corrected:
results[model + 'Corrected'] = corrected
# TODO:
# * check that all files are associated with an existing item
# * check that all files exist within their assetstore and are the
# expected size
# * check that all folders have a valid ancestor tree leading to a
# user or collection
# * check that all folders have the correct baseParentId and
# baseParentType
# * check that all groups contain valid users
# * check that all resources validate
# * for filesystem assetstores, find files that are not tracked.
# * for gridfs assetstores, find chunks that are not tracked.
# * for s3 assetstores, find elements that are not tracked.
return results
def delete_resources(self, event):
user = self.getCurrentUser()
wt_resources = self._filter_resources(event, level=AccessType.WRITE, user=user)
progress = event.info["params"].get("progress", False)
with ProgressContext(
progress,
user=user,
title="Deleting resources",
message="Calculating requirements...",
total=len(wt_resources),
) as ctx:
for obj in wt_resources:
source_path = obj["src_path"]
ctx.update(message="Deleting %s %s" % (obj["kind"], source_path.name))
if obj["kind"] == "folder":
shutil.rmtree(source_path.as_posix())
else:
source_path.unlink()
ctx.update(increment=1)
def moveResources(self, resources, parentType, parentId, progress):
user = self.getCurrentUser()
parent = self._prepareMoveOrCopy(resources, parentType, parentId)
total = sum([len(resources[key]) for key in resources])
with ProgressContext(
progress, user=user, title='Moving resources',
message='Calculating requirements...', total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind, 'move')
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.WRITE, exc=True)
ctx.update(message='Moving %s %s' % (kind, doc.get('name', '')))
if kind == 'item':
if parent['_id'] != doc['folderId']:
model.move(doc, parent)
elif kind == 'folder':
if ((parentType, parent['_id']) !=
(doc['parentCollection'], doc['parentId'])):
model.move(doc, parent, parentType)
ctx.update(increment=1)
def importData(self, assetstore, params):
self.requireParams(('parentId', 'path'), params)
user = self.getCurrentUser()
parentType = params.get('parentType', 'folder')
if parentType not in ('user', 'collection', 'folder'):
raise RestException('Invalid parentType.')
parent = self.model(parentType).load(params['parentId'], force=True, exc=True)
progress = self.boolParam('progress', params, default=False)
client = HdfsClient(
host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False)
path = params['path']
with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx:
try:
self._importData(parentType, parent, assetstore, client, path, ctx, user)
except FileNotFoundException:
raise RestException('File not found: %s.' % path)
def importData(self, assetstore, importPath, destinationId,
destinationType, progress, leafFoldersAsItems,
fileIncludeRegex, fileExcludeRegex):
user = self.getCurrentUser()
parent = ModelImporter.model(destinationType).load(
destinationId, user=user, level=AccessType.ADMIN, exc=True)
with ProgressContext(progress, user=user,
title='Importing data') as ctx:
return self._model.importData(
assetstore,
parent=parent,
parentType=destinationType,
params={
'fileIncludeRegex': fileIncludeRegex,
'fileExcludeRegex': fileExcludeRegex,
'importPath': importPath,
},
progress=ctx,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
def delete(self, params):
"""
Delete a set of resources.
"""
user = self.getCurrentUser()
resources = self._validateResourceSet(params)
total = sum([len(resources[key]) for key in resources])
progress = self.boolParam('progress', params, default=False)
with ProgressContext(progress,
user=user,
title='Deleting resources',
message='Calculating size...') as ctx:
ctx.update(total=total)
current = 0
for kind in resources:
model = self._getResourceModel(kind, 'remove')
for id in resources[kind]:
if (isinstance(model, (acl_mixin.AccessControlMixin,
AccessControlledModel))):
doc = model.load(id=id,
user=user,
level=AccessType.ADMIN)
else:
doc = model.load(id=id)
if not doc:
raise RestException('Resource %s %s not found.' %
(kind, id))
# Don't do a subtree count if we weren't asked for progress
if progress:
subtotal = model.subtreeCount(doc)
if subtotal != 1:
total += model.subtreeCount(doc) - 1
ctx.update(total=total)
model.remove(doc, progress=ctx)
if progress:
current += subtotal
if ctx.progress['data']['current'] != current:
ctx.update(current=current,
message='Deleted ' + kind)
def copy_resources(self, event, path, root, user=None):
wt_resources = self._filter_resources(event, level=AccessType.READ, user=user)
progress = event.info["params"].get("progress", False)
with ProgressContext(
progress,
user=user,
title="Copying resources",
message="Calculating requirements...",
total=len(wt_resources),
) as ctx:
for obj in wt_resources:
source_path = obj["src_path"]
ctx.update(message="Copying %s %s" % (obj["kind"], source_path.name))
if obj["kind"] == "folder":
shutil.copytree(
source_path.as_posix(), (path / source_path.name).as_posix()
)
else:
name = source_path.name
new_path = ensure_unique_path(path, name)
shutil.copy(source_path.as_posix(), new_path.as_posix())
ctx.update(increment=1)
def updateFolderAccess(self, folder, params):
self.requireParams('access', params)
user = self.getCurrentUser()
public = self.boolParam('public', params)
recurse = self.boolParam('recurse', params, default=False)
progress = self.boolParam('progress', params, default=False) and recurse
try:
access = json.loads(params['access'])
except ValueError:
raise RestException('The access parameter must be JSON.')
with ProgressContext(progress, user=user, title='Updating permissions',
message='Calculating progress...') as ctx:
if progress:
ctx.update(total=self.model('folder').subtreeCount(
folder, includeItems=False, user=user,
level=AccessType.ADMIN))
return self.model('folder').setAccessList(
folder, access, save=True, recurse=recurse, user=user,
progress=ctx, setPublic=public)
def copyResources(self, params):
"""
Copy the specified resources to a new parent folder, user, or
collection. Only folder and item resources can be copied with this
function.
"""
user, resources, parent, parentType, progress = \
self._prepareMoveOrCopy(params)
total = len(resources.get('item', []))
if 'folder' in resources:
model = self._getResourceModel('folder')
for id in resources['folder']:
folder = model.load(id=id, user=user, level=AccessType.READ)
if folder:
total += model.subtreeCount(folder)
with ProgressContext(progress,
user=user,
title='Copying resources',
message='Calculating requirements...',
total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind)
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.READ)
if not doc:
raise RestException('Resource not found. No %s with '
'id %s' % (kind, id))
ctx.update(message='Copying %s %s' %
(kind, doc.get('name', '')))
if kind == 'item':
model.copyItem(doc, folder=parent, creator=user)
ctx.update(increment=1)
elif kind == 'folder':
model.copyFolder(doc,
parent=parent,
parentType=parentType,
creator=user,
progress=ctx)
def copyFolder(self, folder, params):
user = self.getCurrentUser()
parentType = params.get('parentType', folder['parentCollection'])
if 'parentId' in params:
parentId = params.get('parentId', folder['parentId'])
parent = self.model(parentType).load(
id=parentId, user=user, level=AccessType.WRITE, exc=True)
else:
parent = None
name = params.get('name', None)
description = params.get('description', None)
public = params.get('public', None)
progress = self.boolParam('progress', params, default=False)
with ProgressContext(progress, user=self.getCurrentUser(),
title='Copying folder %s' % folder['name'],
message='Calculating folder size...') as ctx:
# Don't do the subtree count if we weren't asked for progress
if progress:
ctx.update(total=self.model('folder').subtreeCount(folder))
return self.model('folder').copyFolder(
folder, creator=user, name=name, parentType=parentType,
parent=parent, description=description, public=public,
progress=ctx)
def copyResources(self, resources, parentType, parentId, progress):
user = self.getCurrentUser()
parent = self._prepareMoveOrCopy(resources, parentType, parentId)
total = len(resources.get('item', []))
if 'folder' in resources:
model = self._getResourceModel('folder')
for id in resources['folder']:
folder = model.load(id=id, user=user, level=AccessType.READ, exc=True)
total += model.subtreeCount(folder)
with ProgressContext(
progress, user=user, title='Copying resources',
message='Calculating requirements...', total=total) as ctx:
for kind in resources:
model = self._getResourceModel(kind)
for id in resources[kind]:
doc = model.load(id=id, user=user, level=AccessType.READ, exc=True)
ctx.update(message='Copying %s %s' % (kind, doc.get('name', '')))
if kind == 'item':
model.copyItem(doc, folder=parent, creator=user)
ctx.update(increment=1)
elif kind == 'folder':
model.copyFolder(
doc, parent=parent, parentType=parentType, creator=user, progress=ctx)
def _progressContext(self, folder, title):
user = self.getCurrentUser()
total = self.model('folder').subtreeCount(folder, includeItems=False)
return ProgressContext(True, user=user, total=total, title=title)
def _exportTar(self, assetstore, folder, path, compression, progress):
user = self.getCurrentUser()
adapter = getAssetstoreAdapter(assetstore)
with ProgressContext(progress, user=user, title='Archiving %s' % folder['name']) as ctx:
adapter._exportTar(path, folder, ctx, user, compression)
def _testStream(self, user, token=None):
# Should only work for users or token sessions
resp = self.request(path='/notification/stream', method='GET')
self.assertStatus(resp, 401)
self.assertEqual(resp.json['message'],
'You must be logged in or have a valid auth token.')
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
self.assertStatusOk(resp)
self.assertEqual(self.getBody(resp), '')
# Use a very high rate-limit interval so that we don't fail on slow
# build boxes
with ProgressContext(True,
user=user,
token=token,
title='Test',
total=100,
interval=100) as progress:
progress.update(current=1)
# Rate limiting should make it so we didn't write the immediate
# update within the time interval.
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['type'], 'progress')
self.assertEqual(messages[0]['data']['total'], 100)
self.assertEqual(messages[0]['data']['current'], 0)
self.assertFalse(
ProgressState.isComplete(messages[0]['data']['state']))
# Now use a very short interval to test that we do save changes
progress.interval = 0.01
time.sleep(0.02)
progress.update(current=2)
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['data']['current'], 2)
# If we use a non-numeric value, nothing bad should happen
time.sleep(0.02)
progress.update(current='not_a_number')
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['data']['current'], 'not_a_number')
# Updating the progress without saving and then exiting should
# send the update.
progress.interval = 1000
progress.update(current=3)
# The message should contain a timestamp
self.assertIn('_girderTime', messages[0])
self.assertIsInstance(messages[0]['_girderTime'], int)
# Test that the "since" parameter correctly filters out messages
since = messages[0]['_girderTime'] + 1
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={
'timeout': 0,
'since': since
})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 0)
# Exiting the context manager should flush the most recent update.
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['data']['current'], 3)
# Test a ValidationException within the progress context
try:
with ProgressContext(True,
user=user,
token=token,
title='Test',
total=100):
raise ValidationException('Test Message')
except ValidationException:
pass
# Exiting the context manager should flush the most recent update.
resp = self.request(path='/notification/stream',
method='GET',
user=user,
token=token,
isJson=False,
params={'timeout': 0})
messages = self.getSseMessages(resp)
self.assertEqual(messages[-1]['data']['message'],
'Error: Test Message')
def computeHashes(self, file, progress):
with ProgressContext(progress,
title='Computing hash: %s' % file['name'],
total=file['size'],
user=self.getCurrentUser()) as pc:
return _computeHash(file, progress=pc)
def buildWebCode(self, progress, dev):
user = self.getCurrentUser()
with ProgressContext(progress, user=user, title='Building web client code') as progress:
install.runWebBuild(dev=dev, progress=progress)
def testMoveBetweenAssetstores(self):
folder = six.next(Folder().childFolders(self.admin,
parentType='user',
force=True,
filters={'name': 'Public'}))
resp = self.request(path='/assetstore', method='GET', user=self.admin)
self.assertStatusOk(resp)
fs_assetstore = resp.json[0]
# Clear any old DB data
base.dropGridFSDatabase('girder_test_assetstore_move_assetstore')
params = {
'name': 'New Name',
'type': AssetstoreType.GRIDFS,
'db': 'girder_test_assetstore_move_assetstore'
}
resp = self.request(path='/assetstore',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
gridfs_assetstore = resp.json
# Upload a file - it should go to the fs assetstore
uploadData = 'helloworld'
params = {
'parentType': 'folder',
'parentId': folder['_id'],
'name': 'sample1',
'size': len(uploadData),
'mimeType': 'text/plain'
}
resp = self.request(path='/file',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
upload = resp.json
resp = self.request(path='/file/chunk',
method='POST',
user=self.admin,
body=uploadData,
params={'uploadId': upload['_id']},
type='text/plain')
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], fs_assetstore['_id'])
uploadedFiles = [resp.json]
# Upload it again targetting a different assetstore
params['assetstoreId'] = gridfs_assetstore['_id']
resp = self.request(path='/file',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
upload = resp.json
resp = self.request(path='/file/chunk',
method='POST',
user=self.admin,
body=uploadData,
params={'uploadId': upload['_id']},
type='text/plain')
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], gridfs_assetstore['_id'])
uploadedFiles.append(resp.json)
# Replace the first file, directing the replacement to a different
# assetstore
replaceParams = {
'size': len(uploadData),
'assetstoreId': gridfs_assetstore['_id'],
}
resp = self.request(path='/file/%s/contents' % uploadedFiles[0]['_id'],
method='PUT',
user=self.admin,
params=replaceParams)
self.assertStatusOk(resp)
upload = resp.json
resp = self.request(path='/file/chunk',
method='POST',
user=self.admin,
body=uploadData,
params={'uploadId': upload['_id']},
type='text/plain')
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], gridfs_assetstore['_id'])
uploadedFiles[0] = resp.json
# Move a file from the gridfs assetstore to the filesystem assetstore
resp = self.request(path='/file/%s/move' % uploadedFiles[0]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': fs_assetstore['_id']})
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], fs_assetstore['_id'])
uploadedFiles[0] = resp.json
# Doing it again shouldn't change it.
resp = self.request(path='/file/%s/move' % uploadedFiles[0]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': fs_assetstore['_id']})
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], fs_assetstore['_id'])
uploadedFiles[0] = resp.json
# We should be able to move it back
resp = self.request(path='/file/%s/move' % uploadedFiles[0]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': gridfs_assetstore['_id']})
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], gridfs_assetstore['_id'])
uploadedFiles[0] = resp.json
# Test moving a file of zero length
params['size'] = 0
resp = self.request(path='/file',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
uploadedFiles.append(resp.json)
resp = self.request(path='/file/%s/move' % uploadedFiles[2]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': fs_assetstore['_id']})
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], fs_assetstore['_id'])
uploadedFiles[2] = resp.json
# Test preventing the move via an event
def stopMove(event):
event.preventDefault()
events.bind('model.upload.movefile', 'assetstore_test', stopMove)
try:
resp = self.request(path='/file/%s/move' % uploadedFiles[0]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': fs_assetstore['_id']},
isJson=False)
self.assertFalse('Move should have been prevented')
except AssertionError as exc:
self.assertIn('could not be moved to assetstore', str(exc))
events.unbind('model.upload.movefile', 'assetstore_test')
# Test files big enough to be multi-chunk
chunkSize = Upload()._getChunkSize()
data = io.BytesIO(b' ' * chunkSize * 2)
uploadedFiles.append(Upload().uploadFromFile(data,
chunkSize * 2,
'sample',
parentType='folder',
parent=folder,
assetstore=fs_assetstore))
resp = self.request(path='/file/%s/move' % uploadedFiles[3]['_id'],
method='PUT',
user=self.admin,
params={'assetstoreId': gridfs_assetstore['_id']})
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], gridfs_assetstore['_id'])
uploadedFiles[3] = resp.json
# Test progress
size = chunkSize * 2
data = io.BytesIO(b' ' * size)
upload = Upload().uploadFromFile(data,
size,
'progress',
parentType='folder',
parent=folder,
assetstore=fs_assetstore)
params = {'assetstoreId': gridfs_assetstore['_id'], 'progress': True}
resp = self.request(path='/file/%s/move' % upload['_id'],
method='PUT',
user=self.admin,
params=params)
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], gridfs_assetstore['_id'])
resp = self.request(path='/notification/stream',
method='GET',
user=self.admin,
isJson=False,
params={'timeout': 1})
messages = self.getSseMessages(resp)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['type'], 'progress')
self.assertEqual(messages[0]['data']['current'], size)
# Test moving imported file
# Create assetstore to import file into
params = {
'name': 'ImportTest',
'type': AssetstoreType.FILESYSTEM,
'root': os.path.join(fs_assetstore['root'], 'import')
}
resp = self.request(path='/assetstore',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
import_assetstore = resp.json
# Import file
params = {
'importPath':
os.path.join(ROOT_DIR, 'tests', 'cases', 'py_client', 'testdata',
'world.txt'),
'destinationType':
'folder',
}
Assetstore().importData(import_assetstore,
parent=folder,
parentType='folder',
params=params,
progress=ProgressContext(False),
user=self.admin,
leafFoldersAsItems=False)
file = path_util.lookUpPath('/user/admin/Public/world.txt/world.txt',
self.admin)['document']
# Move file
params = {
'assetstoreId': fs_assetstore['_id'],
}
resp = self.request(path='/file/%s/move' % file['_id'],
method='PUT',
user=self.admin,
params=params)
self.assertStatusOk(resp)
self.assertEqual(resp.json['assetstoreId'], fs_assetstore['_id'])
# Check that we can still download the file
resp = self.request(path='/file/%s/download' % file['_id'],
user=self.admin,
isJson=False)
self.assertStatusOk(resp)