def untrackedUploads(self, action='list', assetstoreId=None):
"""
List or discard any uploads that an assetstore knows about but that our
database doesn't have in it.
:param action: 'delete' to discard the untracked uploads, anything else
to just return with a list of them.
:type action: str
:param assetstoreId: if present, only include untracked items from the
specified assetstore.
:type assetstoreId: str
:returns: a list of items that were removed or could be removed.
"""
from girderformindlogger.models.assetstore import Assetstore
from girderformindlogger.utility import assetstore_utilities
results = []
knownUploads = list(self.list())
# Iterate through all assetstores
for assetstore in Assetstore().list():
if assetstoreId and assetstoreId != assetstore['_id']:
continue
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
results.extend(
adapter.untrackedUploads(knownUploads,
delete=(action == 'delete')))
except ValidationException:
# this assetstore is currently unreachable, so skip it
pass
return results
def requestOffset(self, upload):
"""
Requests the offset that should be used to resume uploading. This
makes the request from the assetstore adapter.
"""
from girderformindlogger.models.assetstore import Assetstore
from girderformindlogger.utility import assetstore_utilities
assetstore = Assetstore().load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
return adapter.requestOffset(upload)
def __init__(self):
super(Assetstore, self).__init__()
self.resourceName = 'assetstore'
self._model = AssetstoreModel()
self.route('GET', (), self.find)
self.route('GET', (':id', ), self.getAssetstore)
self.route('POST', (), self.createAssetstore)
self.route('POST', (':id', 'import'), self.importData)
self.route('PUT', (':id', ), self.updateAssetstore)
self.route('DELETE', (':id', ), self.deleteAssetstore)
self.route('GET', (':id', 'files'), self.getAssetstoreFiles)
def _getAssetstoreModel(self, file):
from girderformindlogger.models.assetstore import Assetstore
if file.get('assetstoreType'):
try:
if isinstance(file['assetstoreType'], six.string_types):
return ModelImporter.model(file['assetstoreType'])
else:
return ModelImporter.model(*file['assetstoreType'])
except Exception:
raise ValidationException('Invalid assetstore type: %s.' %
(file['assetstoreType'], ))
else:
return Assetstore()
def fsAssetstore(db, request):
"""
Require a filesystem assetstore. Its location will be derived from the test function name.
"""
from girderformindlogger.constants import ROOT_DIR
from girderformindlogger.models.assetstore import Assetstore
name = _uid(request.node)
path = os.path.join(ROOT_DIR, 'tests', 'assetstore', name)
if os.path.isdir(path):
shutil.rmtree(path)
yield Assetstore().createFilesystemAssetstore(name=name, root=path)
if os.path.isdir(path):
shutil.rmtree(path)
def testAssetstorePolicy(self):
"""
Test assetstore policies for a user and a collection.
"""
# We want three assetstores for testing, one of which is unreachable.
# We already have one, which is the current assetstore.
base.dropGridFSDatabase('girder_test_user_quota_assetstore')
params = {
'name': 'Non-current Store',
'type': AssetstoreType.GRIDFS,
'db': 'girder_test_user_quota_assetstore'
}
resp = self.request(path='/assetstore',
method='POST',
user=self.admin,
params=params)
self.assertStatusOk(resp)
# Create a broken assetstore. (Must bypass validation since it should
# not let us create an assetstore in a broken state).
Assetstore().save(
{
'name': 'Broken Store',
'type': AssetstoreType.FILESYSTEM,
'root': '/dev/null',
'created': datetime.datetime.utcnow()
},
validate=False)
# Now get the assetstores and save their ids for later
resp = self.request(path='/assetstore',
method='GET',
user=self.admin,
params={'sort': 'created'})
self.assertStatusOk(resp)
assetstores = resp.json
self.assertEqual(len(assetstores), 3)
self.currentAssetstore = assetstores[0]
self.alternateAssetstore = assetstores[1]
self.brokenAssetstore = assetstores[2]
self._testAssetstores('user', self.user, self.admin)
self._testAssetstores('collection', self.collection, self.admin)
def handleChunk(self, upload, chunk, filter=False, user=None):
"""
When a chunk is uploaded, this should be called to process the chunk.
If this is the final chunk of the upload, this method will finalize
the upload automatically.
This method will return EITHER an upload or a file document. If this
is the final chunk of the upload, the upload is finalized and the created
file document is returned. Otherwise, it returns the upload document
with the relevant fields modified.
:param upload: The upload document to update.
:type upload: dict
:param chunk: The file object representing the chunk that was uploaded.
:type chunk: file
:param filter: Whether the model should be filtered. Only affects
behavior when returning a file model, not the upload model.
:type filter: bool
:param user: The current user. Only affects behavior if filter=True.
:type user: dict or None
"""
from girderformindlogger.models.assetstore import Assetstore
from girderformindlogger.models.file import File
from girderformindlogger.utility import assetstore_utilities
assetstore = Assetstore().load(upload['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
upload = adapter.uploadChunk(upload, chunk)
if '_id' in upload or upload['received'] != upload['size']:
upload = self.save(upload)
# If upload is finished, we finalize it
if upload['received'] == upload['size']:
file = self.finalizeUpload(upload, assetstore)
if filter:
return File().filter(file, user=user)
else:
return file
else:
return upload
def updateFileContents(self, file, size, reference, assetstoreId):
user = self.getCurrentUser()
assetstore = None
if assetstoreId:
self.requireAdmin(
user,
message=
'You must be an admin to select a destination assetstore.')
assetstore = Assetstore().load(assetstoreId)
# Create a new upload record into the existing file
upload = Upload().createUploadToFile(file=file,
user=user,
size=size,
reference=reference,
assetstore=assetstore)
if upload['size'] > 0:
return upload
else:
return self._model.filter(Upload().finalizeUpload(upload), user)
def cancelUpload(self, upload):
"""
Discard an upload that is in progress. This asks the assetstore to
discard the data, then removes the item from the upload database.
:param upload: The upload document to remove.
:type upload: dict
"""
from girderformindlogger.models.assetstore import Assetstore
from girderformindlogger.utility import assetstore_utilities
assetstore = Assetstore().load(upload['assetstoreId'])
# If the assetstore was deleted, the upload may still be in our
# database
if assetstore:
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
try:
adapter.cancelUpload(upload)
except ValidationException:
# this assetstore is currently unreachable, so skip it
pass
if '_id' in upload:
self.remove(upload)
def _checkAssetstore(self, assetstoreSpec):
"""
Check is a specified assetstore is available.
:param assetstoreSpec: None for use current assetstore, 'none' to
disallow the assetstore, or an assetstore ID to
check if that assetstore exists and is nominally
available.
:returns: None to use the current assetstore, False to indicate no
assetstore is allowed, or an assetstore document of an
allowed assetstore.
"""
if assetstoreSpec is None:
return None
if assetstoreSpec == 'none':
return False
assetstore = Assetstore().load(id=assetstoreSpec)
if not assetstore:
return False
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
if getattr(adapter, 'unavailable', False):
return False
return assetstore
def getTargetAssetstore(self, modelType, resource, assetstore=None):
"""
Get the assetstore for a particular target resource, i.e. where new
data within the resource should be stored. In Girder core, this is
always just the current assetstore, but plugins may override this
behavior to allow for more granular assetstore selection.
:param modelType: the type of the resource that will be stored.
:param resource: the resource to be stored.
:param assetstore: if specified, the preferred assetstore where the
resource should be located. This may be overridden.
:returns: the selected assetstore.
"""
from girderformindlogger.models.assetstore import Assetstore
eventParams = {'model': modelType, 'resource': resource}
event = events.trigger('model.upload.assetstore', eventParams)
if event.responses:
assetstore = event.responses[-1]
elif not assetstore:
assetstore = Assetstore().getCurrent()
return assetstore
def finalizeUpload(self, upload, assetstore=None):
"""
This should only be called manually in the case of creating an
empty file, i.e. one that has no chunks.
:param upload: The upload document.
:type upload: dict
:param assetstore: If known, the containing assetstore for the upload.
:type assetstore: dict
:returns: The file object that was created.
"""
from girderformindlogger.models.assetstore import Assetstore
from girderformindlogger.models.file import File
from girderformindlogger.models.item import Item
from girderformindlogger.utility import assetstore_utilities
events.trigger('model.upload.finalize', upload)
if assetstore is None:
assetstore = Assetstore().load(upload['assetstoreId'])
if 'fileId' in upload: # Updating an existing file's contents
file = File().load(upload['fileId'], force=True)
# Delete the previous file contents from the containing assetstore
assetstore_utilities.getAssetstoreAdapter(Assetstore().load(
file['assetstoreId'])).deleteFile(file)
item = Item().load(file['itemId'], force=True)
File().propagateSizeChange(item, upload['size'] - file['size'])
# Update file info
file['creatorId'] = upload['userId']
file['created'] = datetime.datetime.utcnow()
file['assetstoreId'] = assetstore['_id']
file['size'] = upload['size']
# If the file was previously imported, it is no longer.
if file.get('imported'):
file['imported'] = False
else: # Creating a new file record
if upload.get('attachParent'):
item = None
elif upload['parentType'] == 'folder':
# Create a new item with the name of the file.
item = Item().createItem(name=upload['name'],
creator={'_id': upload['userId']},
folder={'_id': upload['parentId']})
elif upload['parentType'] == 'item':
item = Item().load(id=upload['parentId'], force=True)
else:
item = None
file = File().createFile(item=item,
name=upload['name'],
size=upload['size'],
creator={'_id': upload['userId']},
assetstore=assetstore,
mimeType=upload['mimeType'],
saveFile=False)
if upload.get('attachParent'):
if upload['parentType'] and upload['parentId']:
file['attachedToType'] = upload['parentType']
file['attachedToId'] = upload['parentId']
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
file = adapter.finalizeUpload(upload, file)
event_document = {'file': file, 'upload': upload}
events.trigger('model.file.finalizeUpload.before', event_document)
file = File().save(file)
events.trigger('model.file.finalizeUpload.after', event_document)
if '_id' in upload:
self.remove(upload)
logger.info('Upload complete. Upload=%s File=%s User=%s' %
(upload['_id'], file['_id'], upload['userId']))
# Add an async event for handlers that wish to process this file.
eventParams = {
'file': file,
'assetstore': assetstore,
'currentToken': rest.getCurrentToken(),
'currentUser': rest.getCurrentUser()
}
if 'reference' in upload:
eventParams['reference'] = upload['reference']
events.daemon.trigger('data.process', eventParams)
return file
def initUpload(self, parentType, parentId, name, size, mimeType, linkUrl,
reference, assetstoreId):
"""
Before any bytes of the actual file are sent, a request should be made
to initialize the upload. This creates the temporary record of the
forthcoming upload that will be passed in chunks to the readChunk
method. If you pass a "linkUrl" parameter, it will make a link file
in the designated parent.
"""
user = self.getCurrentUser()
parent = ModelImporter.model(parentType).load(id=parentId,
user=user,
level=AccessType.WRITE,
exc=True)
if linkUrl is not None:
return self._model.filter(
self._model.createLinkFile(url=linkUrl,
parent=parent,
name=name,
parentType=parentType,
creator=user,
size=size,
mimeType=mimeType), user)
else:
self.requireParams({'size': size})
assetstore = None
if assetstoreId:
self.requireAdmin(
user,
message=
'You must be an admin to select a destination assetstore.')
assetstore = Assetstore().load(assetstoreId)
chunk = None
if size > 0 and cherrypy.request.headers.get('Content-Length'):
ct = cherrypy.request.body.content_type.value
if (ct not in cherrypy.request.body.processors and ct.split(
'/', 1)[0] not in cherrypy.request.body.processors):
chunk = RequestBodyStream(cherrypy.request.body)
if chunk is not None and chunk.getSize() <= 0:
chunk = None
try:
# TODO: This can be made more efficient by adding
# save=chunk is None
# to the createUpload call parameters. However, since this is
# a breaking change, that should be deferred until a major
# version upgrade.
upload = Upload().createUpload(user=user,
name=name,
parentType=parentType,
parent=parent,
size=size,
mimeType=mimeType,
reference=reference,
assetstore=assetstore)
except OSError as exc:
if exc.errno == errno.EACCES:
raise GirderException(
'Failed to create upload.',
'girderformindlogger.api.v1.file.create-upload-failed')
raise
if upload['size'] > 0:
if chunk:
return Upload().handleChunk(upload,
chunk,
filter=True,
user=user)
return upload
else:
return self._model.filter(Upload().finalizeUpload(upload),
user)
class Assetstore(Resource):
"""
API Endpoint for managing assetstores. Requires admin privileges.
"""
def __init__(self):
super(Assetstore, self).__init__()
self.resourceName = 'assetstore'
self._model = AssetstoreModel()
self.route('GET', (), self.find)
self.route('GET', (':id', ), self.getAssetstore)
self.route('POST', (), self.createAssetstore)
self.route('POST', (':id', 'import'), self.importData)
self.route('PUT', (':id', ), self.updateAssetstore)
self.route('DELETE', (':id', ), self.deleteAssetstore)
self.route('GET', (':id', 'files'), self.getAssetstoreFiles)
@access.admin
@autoDescribeRoute(
Description('Get information about an assetstore.').modelParam(
'id', model=AssetstoreModel).errorResponse().errorResponse(
'You are not an administrator.', 403))
def getAssetstore(self, assetstore):
self._model.addComputedInfo(assetstore)
return assetstore
@access.admin
@autoDescribeRoute(
Description('List assetstores.').pagingParams(
defaultSort='name').errorResponse().errorResponse(
'You are not an administrator.', 403))
def find(self, limit, offset, sort):
return self._model.list(offset=offset, limit=limit, sort=sort)
@access.admin
@autoDescribeRoute(
Description('Create a new assetstore.').responseClass('Assetstore').
notes('You must be an administrator to call this.').param(
'name', 'Unique name for the assetstore.').param(
'type', 'Type of the assetstore.', dataType='integer').param(
'root',
'Root path on disk (for filesystem type).',
required=False).param(
'perms',
'File creation permissions (for filesystem type).',
required=False).param(
'db',
'Database name (for GridFS type)',
required=False).param(
'mongohost',
'Mongo host URI (for GridFS type)',
required=False).param(
'replicaset',
'Replica set name (for GridFS type)',
required=False).
param('bucket',
'The S3 bucket to store data in (for S3 type).',
required=False).param(
'prefix',
'Optional path prefix within the bucket under which '
'files will be stored (for S3 type).',
required=False,
default='').param(
'accessKeyId',
'The AWS access key ID to use for authentication '
'(for S3 type).',
required=False).param(
'secret',
'The AWS secret key to use for authentication (for '
'S3 type).',
required=False).
param(
'service', 'The S3 service host (for S3 type). Default is '
's3.amazonaws.com. This can be used to specify a protocol and '
'port as well using the form [http[s]://](host domain)[:(port)]. '
'Do not include the bucket name here.',
required=False,
default='').param(
'readOnly',
'If this assetstore is read-only, set this to true.',
required=False,
dataType='boolean',
default=False).param(
'region',
'The AWS region to which the S3 bucket belongs.',
required=False,
default=DEFAULT_REGION).
param(
'inferCredentials',
'The credentials for connecting to S3 will be inferred '
'by Boto rather than explicitly passed. Inferring credentials will '
'ignore accessKeyId and secret.',
dataType='boolean',
required=False).param(
'serverSideEncryption',
'Whether to use S3 SSE to encrypt the objects uploaded to '
'this bucket (for S3 type).',
dataType='boolean',
required=False,
default=False).errorResponse().errorResponse(
'You are not an administrator.', 403))
def createAssetstore(self, name, type, root, perms, db, mongohost,
replicaset, bucket, prefix, accessKeyId, secret,
service, readOnly, region, inferCredentials,
serverSideEncryption):
if type == AssetstoreType.FILESYSTEM:
self.requireParams({'root': root})
return self._model.createFilesystemAssetstore(name=name,
root=root,
perms=perms)
elif type == AssetstoreType.GRIDFS:
self.requireParams({'db': db})
return self._model.createGridFsAssetstore(name=name,
db=db,
mongohost=mongohost,
replicaset=replicaset)
elif type == AssetstoreType.S3:
self.requireParams({'bucket': bucket})
return self._model.createS3Assetstore(
name=name,
bucket=bucket,
prefix=prefix,
secret=secret,
accessKeyId=accessKeyId,
service=service,
readOnly=readOnly,
region=region,
inferCredentials=inferCredentials,
serverSideEncryption=serverSideEncryption)
else:
raise RestException('Invalid type parameter')
@access.admin(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Import existing data into an assetstore.').notes(
'This does not move or copy the existing data, it just creates '
'references to it in the Girder data hierarchy. Deleting '
'those references will not delete the underlying data. This '
'operation is currently only supported for S3 assetstores.').
modelParam('id', model=AssetstoreModel).param(
'importPath', 'Root path within the underlying storage system '
'to import.',
required=False).param(
'destinationId',
'ID of a folder, collection, or user in Girder '
'under which the data will be imported.').param(
'destinationType',
'Type of the destination resource.',
enum=('folder', 'collection', 'user')).param(
'progress',
'Whether to record progress on the import.',
dataType='boolean',
default=False,
required=False).param(
'leafFoldersAsItems',
'Whether folders containing only files should be '
'imported as items.',
dataType='boolean',
required=False,
default=False).param(
'fileIncludeRegex',
'If set, only filenames matching this regular '
'expression will be imported.',
required=False).
param(
'fileExcludeRegex',
'If set, only filenames that do not match this regular '
'expression will be imported. If a file matches both the include and exclude regex, '
'it will be excluded.',
required=False).errorResponse().errorResponse(
'You are not an administrator.', 403))
def importData(self, assetstore, importPath, destinationId,
destinationType, progress, leafFoldersAsItems,
fileIncludeRegex, fileExcludeRegex):
user = self.getCurrentUser()
parent = ModelImporter.model(destinationType).load(
destinationId, user=user, level=AccessType.ADMIN, exc=True)
with ProgressContext(progress, user=user,
title='Importing data') as ctx:
return self._model.importData(
assetstore,
parent=parent,
parentType=destinationType,
params={
'fileIncludeRegex': fileIncludeRegex,
'fileExcludeRegex': fileExcludeRegex,
'importPath': importPath,
},
progress=ctx,
user=user,
leafFoldersAsItems=leafFoldersAsItems)
@access.admin
@autoDescribeRoute(
Description('Update an existing assetstore.').responseClass(
'Assetstore').modelParam('id', model=AssetstoreModel).param(
'name', 'Unique name for the assetstore.', strip=True).param(
'root',
'Root path on disk (for Filesystem type)',
required=False).param(
'perms',
'File creation permissions (for Filesystem type)',
required=False).param(
'db',
'Database name (for GridFS type)',
required=False).param(
'mongohost',
'Mongo host URI (for GridFS type)',
required=False).param(
'replicaset',
'Replica set name (for GridFS type)',
required=False).
param('bucket',
'The S3 bucket to store data in (for S3 type).',
required=False).param(
'prefix',
'Optional path prefix within the bucket under which '
'files will be stored (for S3 type).',
required=False,
default='').param(
'accessKeyId',
'The AWS access key ID to use for authentication '
'(for S3 type).',
required=False).param(
'secret',
'The AWS secret key to use for authentication (for '
'S3 type).',
required=False).
param(
'service', 'The S3 service host (for S3 type). Default is '
's3.amazonaws.com. This can be used to specify a protocol and '
'port as well using the form [http[s]://](host domain)[:(port)]. '
'Do not include the bucket name here.',
required=False,
default='').param(
'readOnly',
'If this assetstore is read-only, set this to true.',
required=False,
dataType='boolean').param(
'region',
'The AWS region to which the S3 bucket belongs.',
required=False,
default=DEFAULT_REGION).param(
'current',
'Whether this is the current assetstore',
dataType='boolean').
param(
'inferCredentials',
'The credentials for connecting to S3 will be inferred '
'by Boto rather than explicitly passed. Inferring credentials will '
'ignore accessKeyId and secret.',
dataType='boolean',
required=False).param(
'serverSideEncryption',
'Whether to use S3 SSE to encrypt the objects uploaded to '
'this bucket (for S3 type).',
dataType='boolean',
required=False,
default=False).errorResponse().errorResponse(
'You are not an administrator.', 403))
def updateAssetstore(self, assetstore, name, root, perms, db, mongohost,
replicaset, bucket, prefix, accessKeyId, secret,
service, readOnly, region, current, inferCredentials,
serverSideEncryption, params):
assetstore['name'] = name
assetstore['current'] = current
if assetstore['type'] == AssetstoreType.FILESYSTEM:
self.requireParams({'root': root})
assetstore['root'] = root
if perms is not None:
assetstore['perms'] = perms
elif assetstore['type'] == AssetstoreType.GRIDFS:
self.requireParams({'db': db})
assetstore['db'] = db
if mongohost is not None:
assetstore['mongohost'] = mongohost
if replicaset is not None:
assetstore['replicaset'] = replicaset
elif assetstore['type'] == AssetstoreType.S3:
self.requireParams({'bucket': bucket})
assetstore['bucket'] = bucket
assetstore['prefix'] = prefix
assetstore['accessKeyId'] = accessKeyId
assetstore['secret'] = secret
assetstore['service'] = service
assetstore['region'] = region
assetstore['inferCredentials'] = inferCredentials
assetstore['serverSideEncryption'] = serverSideEncryption
if readOnly is not None:
assetstore['readOnly'] = readOnly
else:
event = events.trigger('assetstore.update',
info={
'assetstore':
assetstore,
'params':
dict(name=name,
current=current,
readOnly=readOnly,
root=root,
perms=perms,
db=db,
mongohost=mongohost,
replicaset=replicaset,
bucket=bucket,
prefix=prefix,
accessKeyId=accessKeyId,
secret=secret,
service=service,
region=region,
**params)
})
if event.defaultPrevented:
return
return self._model.save(assetstore)
@access.admin
@autoDescribeRoute(
Description('Delete an assetstore.').notes(
'This will fail if there are any files in the assetstore.').
modelParam('id', model=AssetstoreModel).errorResponse(
('A parameter was invalid.',
'The assetstore is not empty.')).errorResponse(
'You are not an administrator.', 403))
def deleteAssetstore(self, assetstore):
self._model.remove(assetstore)
return {'message': 'Deleted assetstore %s.' % assetstore['name']}
@access.admin
@autoDescribeRoute(
Description(
'Get a list of files controlled by an assetstore.').modelParam(
'id', model=AssetstoreModel).pagingParams(
defaultSort='_id').errorResponse().errorResponse(
'You are not an administrator.', 403))
def getAssetstoreFiles(self, assetstore, limit, offset, sort):
return File().find(query={'assetstoreId': assetstore['_id']},
offset=offset,
limit=limit,
sort=sort)