def testGridFSShardingAssetstoreUpload(self):
verbose = 0
if 'REPLICASET' in os.environ.get('EXTRADEBUG', '').split():
verbose = 2
# Starting the sharding service takes time
rscfg = mongo_replicaset.makeConfig(port=27073, shard=True, sharddb=None)
mongo_replicaset.startMongoReplicaSet(rscfg, verbose=verbose)
# Clear the assetstore database and create a GridFS assetstore
Assetstore().remove(Assetstore().getCurrent())
self.assetstore = Assetstore().createGridFsAssetstore(
name='Test', db='girder_assetstore_shard_upload_test',
mongohost='mongodb://127.0.0.1:27073', shard='auto')
self._testUpload()
# Verify that we have successfully sharded the collection
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
stat = adapter.chunkColl.database.command('collstats', adapter.chunkColl.name)
self.assertTrue(bool(stat['sharded']))
# Although we have asked for multiple shards, the chunks may all be on
# one shard. Make sure at least one shard is reported.
self.assertGreaterEqual(len(stat['shards']), 1)
# Asking for the same database again should also report sharding. Use
# a slightly differt URI to ensure that the sharding is checked anew.
assetstore = Assetstore().createGridFsAssetstore(
name='Test 2', db='girder_assetstore_shard_upload_test',
mongohost='mongodb://127.0.0.1:27073/?', shard='auto')
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
stat = adapter.chunkColl.database.command('collstats', adapter.chunkColl.name)
self.assertTrue(bool(stat['sharded']))
mongo_replicaset.stopMongoReplicaSet(rscfg)
def finalizeUpload(self, upload, assetstore=None):
"""
This should only be called manually in the case of creating an
empty file, i.e. one that has no chunks.
"""
events.trigger('model.upload.finalize', upload)
if assetstore is None:
assetstore = self.model('assetstore').load(upload['assetstoreId'])
if 'fileId' in upload: # Updating an existing file's contents
file = self.model('file').load(upload['fileId'], force=True)
# Delete the previous file contents from the containing assetstore
assetstore_utilities.getAssetstoreAdapter(
self.model('assetstore').load(
file['assetstoreId'])).deleteFile(file)
item = self.model('item').load(file['itemId'], force=True)
self.model('file').propagateSizeChange(
item, upload['size'] - file['size'])
# Update file info
file['creatorId'] = upload['userId']
file['created'] = datetime.datetime.utcnow()
file['assetstoreId'] = assetstore['_id']
file['size'] = upload['size']
else: # Creating a new file record
if upload['parentType'] == 'folder':
# Create a new item with the name of the file.
item = self.model('item').createItem(
name=upload['name'], creator={'_id': upload['userId']},
folder={'_id': upload['parentId']})
elif upload['parentType'] == 'item':
item = self.model('item').load(
id=upload['parentId'], force=True)
else:
item = None
file = self.model('file').createFile(
item=item, name=upload['name'], size=upload['size'],
creator={'_id': upload['userId']}, assetstore=assetstore,
mimeType=upload['mimeType'], saveFile=False)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
file = adapter.finalizeUpload(upload, file)
self.model('file').save(file)
self.remove(upload)
# Add an async event for handlers that wish to process this file.
events.daemon.trigger('data.process', {
'file': file,
'assetstore': assetstore
})
return file
def finalizeUpload(self, upload, assetstore=None):
"""
This should only be called manually in the case of creating an
empty file, i.e. one that has no chunks.
"""
events.trigger("model.upload.finalize", upload)
if assetstore is None:
assetstore = self.model("assetstore").load(upload["assetstoreId"])
if "fileId" in upload: # Updating an existing file's contents
file = self.model("file").load(upload["fileId"])
# Delete the previous file contents from the containing assetstore
assetstore_utilities.getAssetstoreAdapter(self.model("assetstore").load(file["assetstoreId"])).deleteFile(
file
)
item = self.model("item").load(file["itemId"], force=True)
self.model("file").propagateSizeChange(item, upload["size"] - file["size"])
# Update file info
file["creatorId"] = upload["userId"]
file["created"] = datetime.datetime.utcnow()
file["assetstoreId"] = assetstore["_id"]
file["size"] = upload["size"]
else: # Creating a new file record
if upload["parentType"] == "folder":
# Create a new item with the name of the file.
item = self.model("item").createItem(
name=upload["name"], creator={"_id": upload["userId"]}, folder={"_id": upload["parentId"]}
)
else:
item = self.model("item").load(id=upload["parentId"], force=True)
file = self.model("file").createFile(
item=item,
name=upload["name"],
size=upload["size"],
creator={"_id": upload["userId"]},
assetstore=assetstore,
mimeType=upload["mimeType"],
saveFile=False,
)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
file = adapter.finalizeUpload(upload, file)
self.model("file").save(file)
self.remove(upload)
# Add an async event for handlers that wish to process this file.
events.daemon.trigger("data.process", {"file": file, "assetstore": assetstore})
return file
def _importDataFile(self, file, parent, parentType, dbinfo, params):
"""
Validate and finish importing a file.
:param file: The file to store information in. If the parentType is
file, the parent is updated instead.
:param parent: The parent object to import into.
:param parentType: The model type of the parent object.
:param dbinfo: a dictionary of database information for the new file.
:param params: Additional parameters required for the import process.
See importData.
:return: the file that was saved.
"""
# Validate the limit parameter
try:
if params.get('limit') not in (None, ''):
params['limit'] = int(params['limit'])
except ValueError:
raise GirderException(
'limit must be empty or an integer')
# Set or replace the database parameters for the file
dbinfo['imported'] = True
for key in ('sort', 'fields', 'filters', 'group', 'format',
'limit'):
dbinfo[key] = params.get(key)
file[DB_INFO_KEY] = dbinfo
# Validate that we can perform queries by trying to download 1 record
# from the file.
# This intentionally encodes extraParameters as JSON. It could pass
# it as a python dictionary or encode it as a url query string, but
# another Girder plugin has expressed a preference for JSON as the de
# facto standard for extraParameters.
downloadFunc = self.downloadFile(
file.copy(), headers=False, extraParameters=json.dumps({
'limit': 1}))
# Test the download without keeping it
for chunk in downloadFunc():
pass
if parentType == 'file':
assetstore_utilities.getAssetstoreAdapter(
Assetstore().load(parent['assetstoreId'])).deleteFile(parent)
for key in ('creatorId', 'created', 'assetstoreId', 'size',
DB_INFO_KEY):
parent[key] = file[key]
file = parent
# Now save the new file
File().save(file)
return file
def createUploadToFile(self, file, user, size):
"""
Creates a new upload record into a file that already exists. This
should be used when updating the contents of a file. Deletes any
previous file content from the assetstore it was in. This will upload
into the current assetstore rather than assetstore the file was
previously contained in.
:param file: The file record to update.
:param user: The user performing this upload.
:param size: The size of the new file contents.
"""
eventParams = {'model': 'file', 'resource': file}
events.trigger('model.upload.assetstore', eventParams)
assetstore = eventParams.get('assetstore',
self.model('assetstore').getCurrent())
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
upload = {
'created': now,
'updated': now,
'userId': user['_id'],
'fileId': file['_id'],
'assetstoreId': assetstore['_id'],
'size': size,
'name': file['name'],
'mimeType': file['mimeType'],
'received': 0
}
upload = adapter.initUpload(upload)
return self.save(upload)
def untrackedUploads(self, action='list', assetstoreId=None):
"""
List or discard any uploads that an assetstore knows about but that our
database doesn't have in it.
:param action: 'delete' to discard the untracked uploads, anything else
to just return with a list of them.
:type action: str
:param assetstoreId: if present, only include untracked items from the
specified assetstore.
:type assetstoreId: str
:returns: a list of items that were removed or could be removed.
"""
results = []
knownUploads = list(self.list())
# Iterate through all assetstores
for assetstore in self.model('assetstore').list():
if assetstoreId and assetstoreId != assetstore['_id']:
continue
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
results.extend(adapter.untrackedUploads(
knownUploads, delete=(action == 'delete')))
except ValidationException:
# this assetstore is currently unreachable, so skip it
pass
return results
def createUploadToFile(self, file, user, size):
"""
Creates a new upload record into a file that already exists. This
should be used when updating the contents of a file. Deletes any
previous file content from the assetstore it was in. This will upload
into the current assetstore rather than assetstore the file was
previously contained in.
:param file: The file record to update.
:param user: The user performing this upload.
:param size: The size of the new file contents.
"""
eventParams = {"model": "file", "resource": file}
events.trigger("model.upload.assetstore", eventParams)
assetstore = eventParams.get("assetstore", self.model("assetstore").getCurrent())
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
upload = {
"created": now,
"updated": now,
"userId": user["_id"],
"fileId": file["_id"],
"assetstoreId": assetstore["_id"],
"size": size,
"name": file["name"],
"mimeType": file["mimeType"],
"received": 0,
}
upload = adapter.initUpload(upload)
return self.save(upload)
def testAdapterConnectorForTable(self):
# Create assetstore
resp = self.request(path='/assetstore', method='POST', user=self.admin,
params=self.dbParams)
self.assertStatusOk(resp)
assetstore1 = resp.json
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore1)
# This also tests the rawdict and rawlist output formats
conn = adapter.getDBConnectorForTable('towns')
query = adapter.queryDatabase(conn, {'filters': [['town', 'BOSTON']], 'format': 'rawdict'})
data = list(query[0]())
self.assertEqual(data[0]['town'], 'BOSTON')
query = adapter.queryDatabase(conn, {
'filters': [['town', 'BOSTON']],
'fields': ['pop2010', 'town'],
'format': 'rawlist'})
data = list(query[0]())
self.assertEqual(data[0][1], 'BOSTON')
conn = adapter.getDBConnectorForTable('public.towns')
query = adapter.queryDatabase(conn, {'filters': [['town', 'BOSTON']], 'format': 'rawdict'})
data = list(query[0]())
self.assertEqual(data[0]['town'], 'BOSTON')
def handleChunk(self, upload, chunk, filter=False, user=None):
"""
When a chunk is uploaded, this should be called to process the chunk.
If this is the final chunk of the upload, this method will finalize
the upload automatically.
This method will return EITHER an upload or a file document. If this
is the final chunk of the upload, the upload is finalized and the created
file document is returned. Otherwise, it returns the upload document
with the relevant fields modified.
:param upload: The upload document to update.
:type upload: dict
:param chunk: The file object representing the chunk that was uploaded.
:type chunk: file
:param filter: Whether the model should be filtered. Only affects
behavior when returning a file model, not the upload model.
:type filter: bool
:param user: The current user. Only affects behavior if filter=True.
:type user: dict or None
"""
assetstore = self.model('assetstore').load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
upload = self.save(adapter.uploadChunk(upload, chunk))
# If upload is finished, we finalize it
if upload['received'] == upload['size']:
file = self.finalizeUpload(upload, assetstore)
if filter:
return self.model('file').filter(file, user=user)
else:
return file
else:
return upload
def copyFile(self, srcFile, creator, item=None):
"""
Copy a file so that we don't need to duplicate stored data.
:param srcFile: The file to copy.
:type srcFile: dict
:param creator: The user copying the file.
:param item: a new item to assign this file to (optional)
:returns: a dict with the new file.
"""
# Copy the source file's dictionary. The individual assetstore
# implementations will need to fix references if they cannot be
# directly duplicated.
file = srcFile.copy()
# Immediately delete the original id so that we get a new one.
del file['_id']
file['copied'] = datetime.datetime.utcnow()
file['copierId'] = creator['_id']
if item:
file['itemId'] = item['_id']
if file.get('assetstoreId'):
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
adapter.copyFile(srcFile, file)
elif file.get('linkUrl'):
file['linkUrl'] = srcFile['linkUrl']
item = self.model('item').load(id=file['itemId'], user=creator,
level=AccessType.WRITE, exc=True)
if 'size' in file:
self.propagateSizeChange(item, file['size'])
return self.save(file)
def download(self, file, offset=0, headers=True, endByte=None,
contentDisposition=None):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call downloadFile on it. If the file is a link
file rather than a file in an assetstore, we redirect to it.
:param file: The file to download.
:param offset: The start byte within the file.
:type offset: int
:param headers: Whether to set headers (i.e. is this an HTTP request
for a single file, or something else).
:type headers: bool
:param endByte: Final byte to download. If ``None``, downloads to the
end of the file.
:type endByte: int or None
:param contentDisposition: Content-Disposition response header
disposition-type value.
:type contentDisposition: str or None
"""
if file.get('assetstoreId'):
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.downloadFile(
file, offset=offset, headers=headers, endByte=endByte,
contentDisposition=contentDisposition)
elif file.get('linkUrl'):
if headers:
raise cherrypy.HTTPRedirect(file['linkUrl'])
else:
def stream():
yield file['linkUrl']
return stream
else: # pragma: no cover
raise Exception('File has no known download mechanism.')
def validate(self, doc):
# Ensure no duplicate names
q = {'name': doc['name']}
if '_id' in doc:
q['_id'] = {'$ne': doc['_id']}
duplicate = self.findOne(q, fields=['_id'])
if duplicate is not None:
raise ValidationException('An assetstore with that name already '
'exists.', 'name')
# Name must not be empty
if not doc['name']:
raise ValidationException('Name must not be empty.', 'name')
# Adapter classes validate each type internally
adapter = assetstore_utilities.getAssetstoreAdapter(doc, instance=False)
adapter.validateInfo(doc)
# If no current assetstore exists yet, set this one as the current.
current = self.findOne({'current': True}, fields=['_id'])
if current is None:
doc['current'] = True
if 'current' not in doc:
doc['current'] = False
# If we are setting this as current, we should unmark all other
# assetstores that have the current flag.
if doc['current'] is True:
self.update({'current': True}, {'$set': {'current': False}})
return doc
def remove(self, assetstore, **kwargs):
"""
Delete an assetstore. If there are any files within this assetstore,
a validation exception is raised.
:param assetstore: The assetstore document to delete.
:type assetstore: dict
"""
from .file import File
files = File().findOne({'assetstoreId': assetstore['_id']})
if files is not None:
raise ValidationException('You may not delete an assetstore that contains files.')
# delete partial uploads before we delete the store.
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
adapter.untrackedUploads([], delete=True)
except ValidationException:
# this assetstore is currently unreachable, so skip this step
pass
# now remove the assetstore
Model.remove(self, assetstore)
# If after removal there is no current assetstore, then pick a
# different assetstore to be the current one.
current = self.findOne({'current': True})
if current is None:
first = self.findOne(sort=[('created', SortDir.DESCENDING)])
if first is not None:
first['current'] = True
self.save(first)
def finalizeUpload(self, upload, assetstore=None):
"""
This should only be called manually in the case of creating an
empty file, i.e. one that has no chunks.
"""
if assetstore is None:
assetstore = self.model('assetstore').load(upload['assetstoreId'])
if upload['parentType'] == 'folder':
# Create a new item with the name of the file.
item = self.model('item').createItem(
name=upload['name'], creator={'_id': upload['userId']},
folder={'_id': upload['parentId']})
else:
# Uploading into an existing item
item = {'_id': upload['parentId']}
file = self.model('file').createFile(
item=item, name=upload['name'], size=upload['size'],
creator={'_id': upload['userId']}, assetstore=assetstore)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
file = adapter.finalizeUpload(upload, file)
self.model('file').save(file)
self.remove(upload)
return file
def _getLargeImagePath(self):
try:
largeImageFileId = self.item['largeImage']['fileId']
# Access control checking should already have been done on
# item, so don't repeat.
# TODO: is it possible that the file is on a different item, so
# do we want to repeat the access check?
largeImageFile = ModelImporter.model('file').load(
largeImageFileId, force=True)
# TODO: can we move some of this logic into Girder core?
assetstore = ModelImporter.model('assetstore').load(
largeImageFile['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
if not isinstance(
adapter,
assetstore_utilities.FilesystemAssetstoreAdapter):
raise TileSourceAssetstoreException(
'Non-filesystem assetstores are not supported')
largeImagePath = adapter.fullPath(largeImageFile)
return largeImagePath
except TileSourceAssetstoreException:
raise
except (KeyError, ValidationException, TileSourceException) as e:
raise TileSourceException(
'No large image file in this item: %s' % e.message)
def _handleZip(self, prereviewFolder, user, zipFile):
Assetstore = self.model('assetstore')
Image = self.model('image', 'isic_archive')
# Get full path of zip file in assetstore
assetstore = Assetstore.getCurrent()
assetstore_adapter = assetstore_utilities.getAssetstoreAdapter(
assetstore)
fullPath = assetstore_adapter.fullPath(zipFile)
with ZipFileOpener(fullPath) as (fileList, fileCount):
with ProgressContext(
on=True,
user=user,
title='Processing "%s"' % zipFile['name'],
total=fileCount,
state=ProgressState.ACTIVE,
current=0) as progress:
for originalFilePath, originalFileRelpath in fileList:
originalFileName = os.path.basename(originalFileRelpath)
progress.update(
increment=1,
message='Extracting "%s"' % originalFileName)
with open(originalFilePath, 'rb') as originalFileStream:
Image.createImage(
imageDataStream=originalFileStream,
imageDataSize=os.path.getsize(originalFilePath),
originalName=originalFileName,
parentFolder=prereviewFolder,
creator=user
)
def createUpload(self, user, name, parentType, parent, size):
"""
Creates a new upload record, and creates its temporary file
that the chunks will be written into. Chunks should then be sent
in order using the _id of the upload document generated by this method.
:param user: The user performing the upload.
:type user: dict
:param name: The name of the file being uploaded.
:type name: str
:param parentType: The type of the parent being uploaded into.
:type parentType: str ('folder' or 'item')
:param parent: The document representing the parent.
:type parentId: dict
:param size: Total size in bytes of the whole file.
:type size: int
:returns: The upload document that was created.
"""
assetstore = self.model('assetstore').getCurrent()
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.now()
upload = {
'created': now,
'updated': now,
'userId': user['_id'],
'parentType': parentType.lower(),
'parentId': parent['_id'],
'assetstoreId': assetstore['_id'],
'size': size,
'name': name,
'received': 0
}
upload = adapter.initUpload(upload)
return self.save(upload)
def createUploadToFile(self, file, user, size, reference=None):
"""
Creates a new upload record into a file that already exists. This
should be used when updating the contents of a file. Deletes any
previous file content from the assetstore it was in. This will upload
into the current assetstore rather than assetstore the file was
previously contained in.
:param file: The file record to update.
:param user: The user performing this upload.
:param size: The size of the new file contents.
:param reference: An optional reference string that will be sent to the
data.process event.
:type reference: str
"""
assetstore = self.getTargetAssetstore('file', file)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
upload = {
'created': now,
'updated': now,
'userId': user['_id'],
'fileId': file['_id'],
'assetstoreId': assetstore['_id'],
'size': size,
'name': file['name'],
'mimeType': file['mimeType'],
'received': 0
}
if reference is not None:
upload['reference'] = reference
upload = adapter.initUpload(upload)
return self.save(upload)
def download(self, file, offset=0, headers=True):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call downloadFile on it.
"""
assetstore = self.model("assetstore").load(file["assetstoreId"])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.downloadFile(file, offset=offset, headers=headers)
def __enter__(self):
assetstore = ModelImporter.model('assetstore').getCurrent()
assetstore_adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
self.temp_dir = tempfile.mkdtemp(dir=assetstore_adapter.tempDir)
except (AttributeError, OSError):
self.temp_dir = tempfile.mkdtemp()
return self.temp_dir
def requestOffset(self, upload):
"""
Requests the offset that should be used to resume uploading. This
makes the request from the assetstore adapter.
"""
assetstore = self.model('assetstore').load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.requestOffset(upload)
def getAssetstoreAdapter(self, file):
"""
Return the assetstore adapter for the given file.
"""
from girder.utility import assetstore_utilities
assetstore = self._getAssetstoreModel(file).load(file['assetstoreId'])
return assetstore_utilities.getAssetstoreAdapter(assetstore)
def getAssetstoreAdapter(self, file):
"""
Return the assetstore adapter for the given file.
"""
from girder.utility import assetstore_utilities
assetstore = self._getAssetstoreModel(file).load(file['assetstoreId'])
return assetstore_utilities.getAssetstoreAdapter(assetstore)
def download(self, file, offset=0):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call downloadFile on it.
"""
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.downloadFile(file, offset)
def requestOffset(self, upload):
"""
Requests the offset that should be used to resume uploading. This
makes the request from the assetstore adapter.
"""
assetstore = self.model('assetstore').load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.requestOffset(upload)
def testFilesystemAssetstoreFindInvalidFiles(self):
# Create several files in the assetstore, some of which point to real
# files on disk and some that don't
folder = six.next(Folder().childFolders(parent=self.admin,
parentType='user',
force=True,
limit=1))
item = Item().createItem('test', self.admin, folder)
path = os.path.join(ROOT_DIR, 'tests', 'cases', 'py_client',
'testdata', 'hello.txt')
real = File().createFile(name='hello.txt',
creator=self.admin,
item=item,
assetstore=self.assetstore,
size=os.path.getsize(path))
real['imported'] = True
real['path'] = path
File().save(real)
fake = File().createFile(name='fake',
creator=self.admin,
item=item,
size=1,
assetstore=self.assetstore)
fake['path'] = 'nonexistent/path/to/file'
fake['sha512'] = '...'
fake = File().save(fake)
fakeImport = File().createFile(name='fakeImport',
creator=self.admin,
item=item,
size=1,
assetstore=self.assetstore)
fakeImport['imported'] = True
fakeImport['path'] = '/nonexistent/path/to/file'
fakeImport = File().save(fakeImport)
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
self.assertTrue(inspect.isgeneratorfunction(adapter.findInvalidFiles))
with ProgressContext(True, user=self.admin, title='test') as p:
for i, info in enumerate(
adapter.findInvalidFiles(progress=p,
filters={'imported': True}), 1):
self.assertEqual(info['reason'], 'missing')
self.assertEqual(info['file']['_id'], fakeImport['_id'])
self.assertEqual(i, 1)
self.assertEqual(p.progress['data']['current'], 2)
self.assertEqual(p.progress['data']['total'], 2)
for i, info in enumerate(adapter.findInvalidFiles(progress=p), 1):
self.assertEqual(info['reason'], 'missing')
self.assertIn(info['file']['_id'],
(fakeImport['_id'], fake['_id']))
self.assertEqual(i, 2)
self.assertEqual(p.progress['data']['current'], 3)
self.assertEqual(p.progress['data']['total'], 3)
def createUpload(self, user, name, parentType, parent, size, mimeType=None,
reference=None, assetstore=None):
"""
Creates a new upload record, and creates its temporary file
that the chunks will be written into. Chunks should then be sent
in order using the _id of the upload document generated by this method.
:param user: The user performing the upload.
:type user: dict
:param name: The name of the file being uploaded.
:type name: str
:param parentType: The type of the parent being uploaded into.
:type parentType: str ('folder' or 'item')
:param parent: The document representing the parent.
:type parentId: dict
:param size: Total size in bytes of the whole file.
:type size: int
:param mimeType: The mimeType of the file.
:type mimeType: str
:param reference: An optional reference string that will be sent to the
data.process event.
:type reference: str
:param assetstore: An optional assetstore to use to store the file. If
unspecified, the current assetstore is used.
:returns: The upload document that was created.
"""
assetstore = self.getTargetAssetstore(parentType, parent, assetstore)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
if not mimeType:
mimeType = 'application/octet-stream'
upload = {
'created': now,
'updated': now,
'assetstoreId': assetstore['_id'],
'size': size,
'name': name,
'mimeType': mimeType,
'received': 0
}
if reference is not None:
upload['reference'] = reference
if parentType and parent:
upload['parentType'] = parentType.lower()
upload['parentId'] = ObjectId(parent['_id'])
else:
upload['parentType'] = None
upload['parentId'] = None
if user:
upload['userId'] = user['_id']
else:
upload['userId'] = None
upload = adapter.initUpload(upload)
return self.save(upload)
def importData(self, assetstore, parent, parentType, params, progress,
user, **kwargs):
"""
Calls the importData method of the underlying assetstore adapter.
"""
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.importData(
parent=parent, parentType=parentType, params=params,
progress=progress, user=user, **kwargs)
def _createTempDirs(self):
assetstore = ModelImporter.model('assetstore').getCurrent()
assetstore_adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
self.temp_dir = tempfile.mkdtemp(dir=assetstore_adapter.tempDir)
self.external_temp_dir = tempfile.mkdtemp(dir=assetstore_adapter.tempDir)
except (AttributeError, OSError):
self.temp_dir = tempfile.mkdtemp()
self.external_temp_dir = tempfile.mkdtemp()
def getItemFilesMapping(self, item, params):
user = self.getCurrentUser()
result = {}
for (path, fileitem) in self.model('item').fileList(
item, user=user, subpath=False, data=False):
assetstore = self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
result[path] = adapter.fullPath(fileitem)
return result
def __enter__(self):
assetstore = Assetstore().getCurrent()
assetstoreAdapter = assetstore_utilities.getAssetstoreAdapter(
assetstore)
try:
self.tempDir = tempfile.mkdtemp(dir=assetstoreAdapter.tempDir)
except (AttributeError, OSError):
self.tempDir = tempfile.mkdtemp()
return self.tempDir
def importData(self, assetstore, parent, parentType, params, progress,
user, **kwargs):
"""
Calls the importData method of the underlying assetstore adapter.
"""
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.importData(
parent=parent, parentType=parentType, params=params,
progress=progress, user=user, **kwargs)
def testFilesystemAssetstoreUpload(self):
self._testUpload()
# Test that a delete during an upload still results in one file
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
size = 101
data = six.BytesIO(b' ' * size)
files = []
files.append(Upload().uploadFromFile(data,
size,
'progress',
parentType='folder',
parent=self.folder,
assetstore=self.assetstore))
fullPath0 = adapter.fullPath(files[0])
conditionRemoveDone = threading.Condition()
conditionInEvent = threading.Condition()
def waitForCondition(*args, **kwargs):
# Single that we are in the event and then wait to be told that
# the delete has occured before returning.
with conditionInEvent:
conditionInEvent.notify()
with conditionRemoveDone:
conditionRemoveDone.wait()
def uploadFileWithWait():
size = 101
data = six.BytesIO(b' ' * size)
files.append(Upload().uploadFromFile(data,
size,
'progress',
parentType='folder',
parent=self.folder,
assetstore=self.assetstore))
events.bind('model.file.finalizeUpload.before', 'waitForCondition',
waitForCondition)
# We create an upload that is bound to an event that waits during the
# finalizeUpload.before event so that the remove will be executed
# during this time.
with conditionInEvent:
t = threading.Thread(target=uploadFileWithWait)
t.start()
conditionInEvent.wait()
self.assertTrue(os.path.exists(fullPath0))
File().remove(files[0])
# We shouldn't actually remove the file here
self.assertTrue(os.path.exists(fullPath0))
with conditionRemoveDone:
conditionRemoveDone.notify()
t.join()
events.unbind('model.file.finalizeUpload.before', 'waitForCondition')
fullPath1 = adapter.fullPath(files[0])
self.assertEqual(fullPath0, fullPath1)
self.assertTrue(os.path.exists(fullPath1))
def remove(self, file):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call deleteFile on it, then delete the file
record from the database.
"""
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
adapter.deleteFile(file)
Model.remove(self, file)
def listItem(self, item, params):
files = []
for fileitem in self.model('item').childFiles(item):
if 'imported' not in fileitem:
store = \
self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
fileitem["path"] = adapter.fullPath(fileitem)
files.append(fileitem)
return {'folders': [], 'files': files}
def _importTar(self, assetstore, folder, path, progress):
importGroupId = Setting().get(WHITELIST_GROUP_SETTING)
if not importGroupId:
raise Exception('Import whitelist group ID is not set')
user = self.getCurrentUser()
if importGroupId not in user['groups']:
raise AccessException('You are not authorized to import tape archive files.')
if assetstore is None:
# This is a reasonable fallback behavior, but we may want something more robust.
# Imported files are weird anyway
assetstore = Assetstore().getCurrent()
if assetstore['type'] != AssetstoreType.FILESYSTEM:
raise Exception('Not a filesystem assetstore: %s' % assetstore['_id'])
with ProgressContext(progress, user=user, title='Importing data') as ctx:
getAssetstoreAdapter(assetstore)._importTar(path, folder, ctx, user)
def testTaleCopy(self):
from girder.plugins.wholetale.models.tale import Tale
from girder.plugins.wholetale.constants import TaleStatus
from girder.plugins.jobs.models.job import Job
from girder.plugins.jobs.constants import JobStatus
tale = Tale().createTale(self.image, [],
creator=self.admin,
public=True)
workspace = Tale().createWorkspace(tale)
# Below workarounds a bug, it will be addressed elsewhere.
workspace = Folder().setPublic(workspace, True, save=True)
adapter = assetstore_utilities.getAssetstoreAdapter(self.ws_assetstore)
size = 101
data = BytesIO(b' ' * size)
files = []
files.append(Upload().uploadFromFile(data,
size,
'file01.txt',
parentType='folder',
parent=workspace,
assetstore=self.ws_assetstore))
fullPath = adapter.fullPath(files[0])
# Create a copy
resp = self.request(path='/tale/{_id}/copy'.format(**tale),
method='POST',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertFalse(new_tale['public'])
self.assertEqual(new_tale['dataSet'], tale['dataSet'])
self.assertEqual(new_tale['copyOfTale'], str(tale['_id']))
self.assertEqual(new_tale['imageId'], str(tale['imageId']))
self.assertEqual(new_tale['creatorId'], str(self.user['_id']))
self.assertEqual(new_tale['status'], TaleStatus.PREPARING)
copied_file_path = re.sub(workspace['name'], new_tale['_id'], fullPath)
job = Job().findOne({'type': 'wholetale.copy_workspace'})
for i in range(10):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.SUCCESS:
break
time.sleep(0.1)
self.assertTrue(os.path.isfile(copied_file_path))
resp = self.request(path='/tale/{_id}'.format(**new_tale),
method='GET',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertEqual(new_tale['status'], TaleStatus.READY)
Tale().remove(new_tale)
Tale().remove(tale)
def getAssetstoreAdapter(self, file):
"""
Return the assetstore adapter for the given file. Return None if the
file has no assetstore.
"""
from girder.utility import assetstore_utilities
if not file.get('assetstoreId'):
return None
assetstore = self._getAssetstoreModel(file).load(file['assetstoreId'])
return assetstore_utilities.getAssetstoreAdapter(assetstore)
def addComputedInfo(self, assetstore):
"""
Add all runtime-computed properties about an assetstore to its document.
:param assetstore: The assetstore object.
:type assetstore: dict
"""
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
assetstore['capacity'] = adapter.capacityInfo()
assetstore['hasFiles'] = (self.model('file').findOne(
{'assetstoreId': assetstore['_id']}) is not None)
def addComputedInfo(self, assetstore):
"""
Add all runtime-computed properties about an assetstore to its document.
:param assetstore: The assetstore object.
:type assetstore: dict
"""
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
assetstore['capacity'] = adapter.capacityInfo()
assetstore['hasFiles'] = (self.model('file').findOne(
{'assetstoreId': assetstore['_id']}) is not None)
def getItemFilesMapping(self, item, params):
user = self.getCurrentUser()
result = {}
for (path, fileitem) in self.model('item').fileList(item,
user=user,
subpath=False,
data=False):
assetstore = self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
result[path] = adapter.fullPath(fileitem)
return result
def requestOffset(self, upload):
"""
Requests the offset that should be used to resume uploading. This
makes the request from the assetstore adapter.
"""
from .assetstore import Assetstore
from girder.utility import assetstore_utilities
assetstore = Assetstore().load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.requestOffset(upload)
def listItem(self, item, params):
files = []
for fileitem in self.model('item').childFiles(item):
if 'imported' not in fileitem and \
fileitem.get('assetstoreId') is not None:
store = \
self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
fileitem["path"] = adapter.fullPath(fileitem)
files.append(fileitem)
return {'folders': [], 'files': files}
def _extractZipPayload():
# TODO: Move assetstore type to wholetale.
assetstore = next((_ for _ in Assetstore().list() if _['type'] == 101),
None)
if assetstore:
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
tempDir = adapter.tempDir
else:
tempDir = None
with tempfile.NamedTemporaryFile(dir=tempDir) as fp:
for chunk in iterBody(2 * 1024**3):
fp.write(chunk)
fp.seek(0)
if not zipfile.is_zipfile(fp):
raise RestException("Provided file is not a zipfile")
with zipfile.ZipFile(fp) as z:
manifest_file = next(
(_ for _ in z.namelist() if _.endswith('manifest.json')),
None)
if not manifest_file:
raise RestException(
"Provided file doesn't contain a Tale manifest")
try:
manifest = json.loads(z.read(manifest_file).decode())
# TODO: is there a better check?
manifest['@id'].startswith('https://data.wholetale.org')
except Exception as e:
raise RestException(
"Couldn't read manifest.json or not a Tale: {}".format(
str(e)))
env_file = next(
(_
for _ in z.namelist() if _.endswith("environment.json")),
None)
try:
environment = json.loads(z.read(env_file).decode())
except Exception as e:
raise RestException(
"Couldn't read environment.json or not a Tale: {}".
format(str(e)))
# Extract files to tmp on workspace assetstore
temp_dir = tempfile.mkdtemp(dir=tempDir)
# In theory malicious content like: abs path for a member, or relative path with
# ../.. etc., is taken care of by zipfile.extractall, but in the end we're still
# unzipping an untrusted content. What could possibly go wrong...?
z.extractall(path=temp_dir)
return temp_dir, manifest_file, manifest, environment
def testGridFSShardingAssetstoreUpload(self):
verbose = 0
if 'REPLICASET' in os.environ.get('EXTRADEBUG', '').split():
verbose = 2
# Starting the sharding service takes time
rscfg = mongo_replicaset.makeConfig(port=27073,
shard=True,
sharddb=None)
mongo_replicaset.startMongoReplicaSet(rscfg, verbose=verbose)
# Clear the assetstore database and create a GridFS assetstore
self.model('assetstore').remove(self.model('assetstore').getCurrent())
self.assetstore = self.model('assetstore').createGridFsAssetstore(
name='Test',
db='girder_assetstore_shard_upload_test',
mongohost='mongodb://127.0.0.1:27073',
shard='auto')
self._testUpload()
# Verify that we have successfully sharded the collection
adapter = assetstore_utilities.getAssetstoreAdapter(self.assetstore)
stat = adapter.chunkColl.database.command('collstats',
adapter.chunkColl.name)
self.assertTrue(bool(stat['sharded']))
# Although we have asked for multiple shards, the chunks may all be on
# one shard. Make sure at least one shard is reported.
self.assertGreaterEqual(len(stat['shards']), 1)
# Asking for the same database again should also report sharding. Use
# a slightly differt URI to ensure that the sharding is checked anew.
assetstore = self.model('assetstore').createGridFsAssetstore(
name='Test 2',
db='girder_assetstore_shard_upload_test',
mongohost='mongodb://127.0.0.1:27073/?',
shard='auto')
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
stat = adapter.chunkColl.database.command('collstats',
adapter.chunkColl.name)
self.assertTrue(bool(stat['sharded']))
mongo_replicaset.stopMongoReplicaSet(rscfg)
def _handle_chunk(self, upload, chunk, filter=False, user=None):
assetstore = Assetstore().load(upload["assetstoreId"])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
upload = adapter.uploadChunk(upload, chunk)
if "_id" in upload or upload["received"] != upload["size"]:
upload = Upload().save(upload)
# If upload is finished, we finalize it
if upload["received"] == upload["size"]:
return self._finalize_upload(upload)
else:
return upload
def listItem(self, item, params, user):
files = []
for fileitem in self.model('item').childFiles(item):
if 'imported' not in fileitem and \
fileitem.get('assetstoreId') is not None:
try:
store = \
self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
fileitem["path"] = adapter.fullPath(fileitem)
except (ValidationException, AttributeError):
pass
files.append(fileitem)
return {'folders': [], 'files': files}
def createUpload(self, user, name, parentType, parent, size, mimeType=None):
"""
Creates a new upload record, and creates its temporary file
that the chunks will be written into. Chunks should then be sent
in order using the _id of the upload document generated by this method.
:param user: The user performing the upload.
:type user: dict
:param name: The name of the file being uploaded.
:type name: str
:param parentType: The type of the parent being uploaded into.
:type parentType: str ('folder' or 'item')
:param parent: The document representing the parent.
:type parentId: dict
:param size: Total size in bytes of the whole file.
:type size: int
:param mimeType: The mimeType of the file.
:type mimeType: str
:returns: The upload document that was created.
"""
assetstore = self.getTargetAssetstore(parentType, parent)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
if not mimeType:
mimeType = 'application/octet-stream'
upload = {
'created': now,
'updated': now,
'assetstoreId': assetstore['_id'],
'size': size,
'name': name,
'mimeType': mimeType,
'received': 0
}
if parentType and parent:
upload['parentType'] = parentType.lower()
upload['parentId'] = ObjectId(parent['_id'])
else:
upload['parentType'] = None
upload['parentId'] = None
if user:
upload['userId'] = user['_id']
else:
upload['userId'] = None
upload = adapter.initUpload(upload)
return self.save(upload)
def updateFile(self, file):
"""
Call this when changing properties of an existing file, such as name
or MIME type. This causes the updated stamp to change, and also alerts
the underlying assetstore adapter that file information has changed.
"""
file['updated'] = datetime.datetime.utcnow()
file = self.save(file)
if file.get('assetstoreId'):
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
adapter.fileUpdated(file)
return file
def testAdapterGetTableList(self):
# Create assetstore
resp = self.request(path='/assetstore', method='POST', user=self.admin,
params=self.dbParams)
self.assertStatusOk(resp)
assetstore1 = resp.json
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore1)
tableList = adapter.getTableList()
tables = tableList[0]['tables']
self.assertIn('towns', [table['name'] for table in tables])
self.assertNotIn('information_schema.tables', [table['name'] for table in tables])
tableList = adapter.getTableList(internalTables=True)
tables = tableList[0]['tables']
self.assertIn('towns', [table['name'] for table in tables])
self.assertIn('information_schema.tables', [table['name'] for table in tables])
def handleChunk(self, upload, chunk, filter=False, user=None):
# traceback.print_stack()
print('(#####)girder/girder/model/upload.py:handleChunk:upload='+str(upload))
"""
When a chunk is uploaded, this should be called to process the chunk.
If this is the final chunk of the upload, this method will finalize
the upload automatically.
This method will return EITHER an upload or a file document. If this
is the final chunk of the upload, the upload is finalized and the created
file document is returned. Otherwise, it returns the upload document
with the relevant fields modified.
:param upload: The upload document to update.
:type upload: dict
:param chunk: The file object representing the chunk that was uploaded.
:type chunk: file
:param filter: Whether the model should be filtered. Only affects
behavior when returning a file model, not the upload model.
:type filter: bool
:param user: The current user. Only affects behavior if filter=True.
:type user: dict or None
"""
from .assetstore import Assetstore
from .file import File
from girder.utility import assetstore_utilities
assetstore = Assetstore().load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
print('(#####)girder/girder/model/upload.py:handleChunk:assetstore='+str(assetstore))
print('(#####)girder/girder/model/upload.py:handleChunk:adapter='+str(adapter))
upload = adapter.uploadChunk(upload, chunk)
if '_id' in upload or upload['received'] != upload['size']:
upload = self.save(upload)
# If upload is finished, we finalize it
print('(#####)girder/girder/model/upload.py:handleChunk:upload[received]='+str(upload['received']))
print('(#####)girder/girder/model/upload.py:handleChunk:upload[size]='+str(upload['size']))
if upload['received'] == upload['size']:
file = self.finalizeUpload(upload, assetstore)
if filter:
return File().filter(file, user=user)
else:
return file
else:
return upload
def list(self, limit=50, offset=0, sort=None):
"""
List all assetstores.
:param limit: Result limit.
:param offset: Result offset.
:param sort: The sort structure to pass to pymongo.
:returns: List of users.
"""
cursor = self.find({}, limit=limit, offset=offset, sort=sort)
assetstores = []
for assetstore in cursor:
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
assetstore['capacity'] = adapter.capacityInfo()
assetstores.append(assetstore)
return assetstores
def addComputedInfo(self, assetstore):
"""
Add all runtime-computed properties about an assetstore to its document.
:param assetstore: The assetstore object.
:type assetstore: dict
"""
from .file import File
try:
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
except NoAssetstoreAdapter:
# If the adapter doesn't exist, use the abstract adapter, since
# this will just give the default capacity information
adapter = AbstractAssetstoreAdapter(assetstore)
assetstore['capacity'] = adapter.capacityInfo()
assetstore['hasFiles'] = File().findOne({'assetstoreId': assetstore['_id']}) is not None
def _getValues(self, assetstore, params):
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
conn = adapter.getDBConnectorForTable(params['table'])
queryParams = {
'fields': [{
'func': 'distinct',
'param': [{
'field': params['column']
}],
'reference': 'value',
}],
'limit':
100,
'format':
'rawdict'
}
result = list(adapter.queryDatabase(conn, queryParams)[0]())
return [row['value'] for row in result]
def cancelUpload(self, upload):
"""
Discard an upload that is in progress. This asks the assetstore to
discard the data, then removes the item from the upload database.
:param upload: The upload document to remove.
:type upload: dict
"""
assetstore = self.model('assetstore').load(upload['assetstoreId'])
# If the assetstore was deleted, the upload may still be in our
# database
if assetstore:
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
adapter.cancelUpload(upload)
except ValidationException:
# this assetstore is currently unreachable, so skip it
pass
self.model('upload').remove(upload)
def createUpload(self, user, name, parentType, parent, size, mimeType):
"""
Creates a new upload record, and creates its temporary file
that the chunks will be written into. Chunks should then be sent
in order using the _id of the upload document generated by this method.
:param user: The user performing the upload.
:type user: dict
:param name: The name of the file being uploaded.
:type name: str
:param parentType: The type of the parent being uploaded into.
:type parentType: str ('folder' or 'item')
:param parent: The document representing the parent.
:type parentId: dict
:param size: Total size in bytes of the whole file.
:type size: int
:param mimeType: The mimeType of the file.
:type mimeType: str
:returns: The upload document that was created.
"""
eventParams = {'model': parentType, 'resource': parent}
events.trigger('model.upload.assetstore', eventParams)
assetstore = eventParams.get('assetstore',
self.model('assetstore').getCurrent())
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
if not mimeType:
mimeType = 'application/octet-stream'
upload = {
'created': now,
'updated': now,
'userId': user['_id'],
'parentType': parentType.lower(),
'parentId': ObjectId(parent['_id']),
'assetstoreId': assetstore['_id'],
'size': size,
'name': name,
'mimeType': mimeType,
'received': 0
}
upload = adapter.initUpload(upload)
return self.save(upload)
def createUploadToFile(self,
file,
user,
size,
reference=None,
assetstore=None):
"""
Creates a new upload record into a file that already exists. This
should be used when updating the contents of a file. Deletes any
previous file content from the assetstore it was in. This will upload
into the current assetstore rather than assetstore the file was
previously contained in.
:param file: The file record to update.
:param user: The user performing this upload.
:param size: The size of the new file contents.
:param reference: An optional reference string that will be sent to the
data.process event.
:type reference: str
:param assetstore: An optional assetstore to use to store the file. If
unspecified, the current assetstore is used.
"""
from girder.utility import assetstore_utilities
assetstore = self.getTargetAssetstore('file', file, assetstore)
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
now = datetime.datetime.utcnow()
upload = {
'created': now,
'updated': now,
'userId': user['_id'],
'fileId': file['_id'],
'assetstoreId': assetstore['_id'],
'size': size,
'name': file['name'],
'mimeType': file['mimeType'],
'received': 0
}
if reference is not None:
upload['reference'] = reference
upload = adapter.initUpload(upload)
return self.save(upload)
def download(self,
file,
offset=0,
headers=True,
endByte=None,
contentDisposition=None):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call downloadFile on it. If the file is a link
file rather than a file in an assetstore, we redirect to it.
:param file: The file to download.
:param offset: The start byte within the file.
:type offset: int
:param headers: Whether to set headers (i.e. is this an HTTP request
for a single file, or something else).
:type headers: bool
:param endByte: Final byte to download. If ``None``, downloads to the
end of the file.
:type endByte: int or None
:param contentDisposition: Content-Disposition response header
disposition-type value.
:type contentDisposition: str or None
"""
if file.get('assetstoreId'):
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.downloadFile(file,
offset=offset,
headers=headers,
endByte=endByte,
contentDisposition=contentDisposition)
elif file.get('linkUrl'):
if headers:
raise cherrypy.HTTPRedirect(file['linkUrl'])
else:
def stream():
yield file['linkUrl']
return stream
else: # pragma: no cover
raise Exception('File has no known download mechanism.')
def handleChunk(self, upload, chunk):
"""
When a chunk is uploaded, this should be called to process the chunk.
If this is the final chunk of the upload, this method will finalize
the upload automatically.
:param upload: The upload document to update.
:type upload: dict
:param chunk: The file object representing the chunk that was uploaded.
:type chunk: file
"""
assetstore = self.model('assetstore').load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
upload = self.save(adapter.uploadChunk(upload, chunk))
# If upload is finished, we finalize it
if upload['received'] == upload['size']:
return self.finalizeUpload(upload, assetstore)
else:
return upload
def remove(self, file, updateItemSize=True):
"""
Use the appropriate assetstore adapter for whatever assetstore the
file is stored in, and call deleteFile on it, then delete the file
record from the database.
:param file: The file document to remove.
:param updateItemSize: Whether to update the item size. Only set this
to False if you plan to delete the item and do not care about updating
its size.
"""
if file.get('assetstoreId'):
assetstore = self.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
adapter.deleteFile(file)
item = self.model('item').load(file['itemId'], force=True)
self.propagateSizeChange(item, -file['size'], updateItemSize)
Model.remove(self, file)
