class Run(AbstractVRResource):
root_tale_field = "runsRootId"
def __init__(self):
super().__init__('run', Constants.RUNS_ROOT_DIR_NAME)
self.route('PATCH', (':id', 'status'), self.setStatus)
self.route('GET', (':id', 'status'), self.status)
self.route('POST', (':id', 'start'), self.startRun)
events.bind("rest.put.run/:id.after", "wt_versioning",
self.update_parents)
events.bind("rest.delete.run/:id.before", "wt_versioning",
self.update_parents)
events.bind('jobs.job.update.after', 'wt_versioning',
self.updateRunStatus)
self.model = RunHierarchyModel()
@access.public
@filtermodel('folder')
@autoDescribeRoute(
Description(
'Retrieves the runs root folder for this tale.').modelParam(
'taleId',
'The ID of a tale',
model=Tale,
level=AccessType.READ,
paramType='query',
destName='tale').
errorResponse(
'Access was denied (if current user does not have write access to this '
'tale)', 403))
def getRoot(self, tale: dict) -> dict:
return super().getRoot(tale)
@access.user(TokenScope.DATA_WRITE)
@filtermodel('folder')
@autoDescribeRoute(
Description(
'Rename a run associated with a tale. Returns the renamed run folder'
).modelParam('id',
'The ID of a run',
model=Folder,
level=AccessType.WRITE,
destName='rfolder').param('name',
'The new name',
required=True,
dataType='string',
paramType='query').
errorResponse(
'Access was denied (if current user does not have write access to this '
'tale)', 403).errorResponse('Illegal file name', 400))
def rename(self, rfolder: dict, name: str) -> dict:
return super().rename(rfolder, name)
@access.public
@filtermodel('folder')
@autoDescribeRoute(
Description('Returns a run.').modelParam('id',
'The ID of a run.',
model=Folder,
level=AccessType.READ,
destName='rfolder').
errorResponse(
'Access was denied (if current user does not have read access to the '
'respective run folder.', 403))
def load(self, rfolder: dict) -> dict:
return rfolder
@access.user(TokenScope.DATA_WRITE)
@filtermodel('folder')
@autoDescribeRoute(
Description(
'Creates a new empty run associated with a given version and returns the new '
'run folder. This does not actually start any computation.'
).modelParam('versionId',
'A version to create the run from.',
model=Folder,
level=AccessType.WRITE,
destName='version',
paramType='query').
param('name',
'An optional name for the run. If not specified, a name will be '
'generated from the current date and time.',
required=False,
dataType='string',
paramType='query').param(
'allowRename',
'Allow to modify "name" if object with the same name '
'already exists.',
required=False,
dataType='boolean',
default=False).
errorResponse(
'Access was denied (if current user does not have write access to the tale '
'associated with this version)',
403).errorResponse('Illegal file name', 400))
def create(self,
version: dict,
name: str = None,
allowRename: bool = False) -> dict:
user = self.getCurrentUser()
return self.model.create(version, name, user, allowRename=allowRename)
@access.user(TokenScope.DATA_OWN)
@autoDescribeRoute(
Description('Deletes a run.').modelParam('id',
'The ID of run',
model=Folder,
level=AccessType.ADMIN,
destName='rfolder').
errorResponse(
'Access was denied (if current user does not have write access to this '
'tale)', 403))
def delete(self, rfolder: dict) -> None:
self.model.remove(rfolder, self.getCurrentUser())
@access.public
@filtermodel('folder')
@autoDescribeRoute(
Description('Lists runs.').modelParam(
'taleId',
'The ID of the tale to which the runs belong.',
model=Tale,
level=AccessType.READ,
destName='tale',
paramType="query").pagingParams(defaultSort='created').
errorResponse(
'Access was denied (if current user does not have read access to this '
'tale)', 403))
def list(self, tale: dict, limit, offset, sort):
return super().list(tale,
user=self.getCurrentUser(),
limit=limit,
offset=offset,
sort=sort)
@access.user(TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Check if a run exists.').modelParam('taleId',
'The ID of a tale.',
model=Tale,
level=AccessType.READ,
destName='tale',
paramType='query').
param(
'name',
'Return the folder with this name or nothing if no such folder exists.',
required=False,
dataType='string').
errorResponse(
'Access was denied (if current user does not have read access to this '
'tale)', 403))
def exists(self, tale: dict, name: str):
return super().exists(tale, name)
@access.user(TokenScope.DATA_READ)
@autoDescribeRoute(
Description(
'Returns the status of a run in an object with two fields: status and '
'statusString. The possible values for status, an integer, are 0, 1, 2, 3, 4, '
'5, with statusString being, respectively, UNKNOWN, STARTING, RUNNING, '
'COMPLETED, FAILED, CANCELLED.').modelParam('id',
'The ID of a run.',
model=Folder,
level=AccessType.READ,
destName='rfolder').
errorResponse(
'Access was denied (if current user does not have read access to '
'this run)', 403))
def status(self, rfolder: dict) -> dict:
return self.model.getStatus(rfolder)
@access.user(TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description(
'Sets the status of the run. See the status query endpoint for details about '
'the meaning of the code.').modelParam('id',
'The ID of a run.',
model=Folder,
level=AccessType.WRITE,
destName='rfolder').param(
'status',
'The status code.',
dataType='integer',
required=True).
errorResponse(
'Access was denied (if current user does not have read access to '
'this run)', 403))
def setStatus(self, rfolder: dict, status: Union[int, RunState]) -> None:
self.model.setStatus(rfolder, status)
@access.user
@autoDescribeRoute(
Description('Start the recorded_run job').modelParam(
'id',
'The ID of a run.',
model=Folder,
level=AccessType.WRITE,
destName='run').param(
'entrypoint',
'Entrypoint command for recorded run. Defaults to run.sh',
required=False,
dataType='string',
paramType='query').
errorResponse(
'Access was denied (if current user does not have write access to '
'this run)', 403))
def startRun(self, run, entrypoint):
user = self.getCurrentUser()
if not entrypoint:
entrypoint = "run.sh"
runRoot = Folder().load(run['parentId'],
user=user,
level=AccessType.WRITE)
tale = Tale().load(runRoot['meta']['taleId'],
user=user,
level=AccessType.READ)
resource = {
'type': 'wt_recorded_run',
'tale_id': tale['_id'],
'tale_title': tale['title']
}
token = Token().createToken(user=user, days=0.5)
notification = init_progress(resource, user, 'Recorded run',
'Initializing', RECORDED_RUN_STEP_TOTAL)
rrTask = recorded_run.signature(
args=[str(run['_id']),
str(tale['_id']), entrypoint],
girder_job_other_fields={
'wt_notification_id': str(notification['_id']),
},
girder_client_token=str(token['_id']),
).apply_async()
return Job().filter(rrTask.job, user=user)
def updateRunStatus(self, event):
"""
Event handler that updates the run status based on the recorded_run task.
"""
job = event.info['job']
if job['title'] == 'Recorded Run' and job.get('status') is not None:
status = int(job['status'])
rfolder = Folder().load(job['args'][0], force=True)
# Store the previous status, if present.
previousStatus = rfolder.get(FIELD_STATUS_CODE, -1)
if status == JobStatus.SUCCESS:
rfolder[FIELD_STATUS_CODE] = RunStatus.COMPLETED.code
elif status == JobStatus.ERROR:
rfolder[FIELD_STATUS_CODE] = RunStatus.FAILED.code
elif status in (JobStatus.QUEUED, JobStatus.RUNNING):
rfolder[FIELD_STATUS_CODE] = RunStatus.RUNNING.code
# If the status changed, save the object
if FIELD_STATUS_CODE in rfolder and rfolder[
FIELD_STATUS_CODE] != previousStatus:
Folder().save(rfolder)
class TilesItemResource(ItemResource):
def __init__(self, apiRoot):
# Don't call the parent (Item) constructor, to avoid redefining routes,
# but do call the grandparent (Resource) constructor
super(ItemResource, self).__init__()
self.resourceName = 'item'
apiRoot.item.route('POST', (':itemId', 'tiles'), self.createTiles)
apiRoot.item.route('GET', (':itemId', 'tiles'), self.getTilesInfo)
apiRoot.item.route('DELETE', (':itemId', 'tiles'), self.deleteTiles)
apiRoot.item.route('GET', (':itemId', 'tiles', 'thumbnail'),
self.getTilesThumbnail)
apiRoot.item.route('GET', (':itemId', 'tiles', 'region'),
self.getTilesRegion)
apiRoot.item.route('GET', (':itemId', 'tiles', 'pixel'),
self.getTilesPixel)
apiRoot.item.route('GET', (':itemId', 'tiles', 'histogram'),
self.getHistogram)
apiRoot.item.route('GET', (':itemId', 'tiles', 'zxy', ':z', ':x', ':y'),
self.getTile)
apiRoot.item.route('GET', (':itemId', 'tiles', 'fzxy', ':frame', ':z', ':x', ':y'),
self.getTileWithFrame)
apiRoot.item.route('GET', (':itemId', 'tiles', 'images'),
self.getAssociatedImagesList)
apiRoot.item.route('GET', (':itemId', 'tiles', 'images', ':image'),
self.getAssociatedImage)
apiRoot.item.route('GET', ('test', 'tiles'), self.getTestTilesInfo)
apiRoot.item.route('GET', ('test', 'tiles', 'zxy', ':z', ':x', ':y'),
self.getTestTile)
apiRoot.item.route('GET', (':itemId', 'tiles', 'dzi.dzi'),
self.getDZIInfo)
apiRoot.item.route('GET', (':itemId', 'tiles', 'dzi_files', ':level', ':xandy'),
self.getDZITile)
apiRoot.item.route('GET', (':itemId', 'tiles', 'internal_metadata'),
self.getInternalMetadata)
filter_logging.addLoggingFilter(
'GET (/[^/ ?#]+)*/item/[^/ ?#]+/tiles/zxy(/[^/ ?#]+){3}',
frequency=250)
filter_logging.addLoggingFilter(
'GET (/[^/ ?#]+)*/item/[^/ ?#]+/tiles/dzi_files(/[^/ ?#]+){2}',
frequency=250)
# Cache the model singleton
self.imageItemModel = ImageItem()
@describeRoute(
Description('Create a large image for this item.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('fileId', 'The ID of the source file containing the image. '
'Required if there is more than one file in the item.',
required=False)
.param('notify', 'If a job is required to create the large image, '
'a nofication can be sent when it is complete.',
dataType='boolean', default=True, required=False)
)
@access.user
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
@filtermodel(model='job', plugin='jobs')
def createTiles(self, item, params):
largeImageFileId = params.get('fileId')
if largeImageFileId is None:
files = list(Item().childFiles(item=item, limit=2))
if len(files) == 1:
largeImageFileId = str(files[0]['_id'])
if not largeImageFileId:
raise RestException('Missing "fileId" parameter.')
largeImageFile = File().load(largeImageFileId, force=True, exc=True)
user = self.getCurrentUser()
token = self.getCurrentToken()
try:
return self.imageItemModel.createImageItem(
item, largeImageFile, user, token,
notify=self.boolParam('notify', params, default=True))
except TileGeneralException as e:
raise RestException(e.args[0])
@classmethod
def _parseTestParams(cls, params):
_adjustParams(params)
return cls._parseParams(params, False, [
('minLevel', int),
('maxLevel', int),
('tileWidth', int),
('tileHeight', int),
('sizeX', int),
('sizeY', int),
('fractal', lambda val: val == 'true'),
('encoding', str),
])
@classmethod
def _parseParams(cls, params, keepUnknownParams, typeList):
"""
Given a dictionary of parameters, check that a list of parameters are
valid data types. The parameters within the list are validated and
copied to a dictionary by themselves.
:param params: the dictionary of parameters to validate.
:param keepUnknownParams: True to copy all parameters, not just those
in the typeList. The parameters in the typeList are still
validated.
:param typeList: a list of tuples of the form (key, dataType, [outkey1,
[outkey2]]). If output keys are used, the original key is renamed
to the the output key. If two output keys are specified, the
original key is renamed to outkey2 and placed in a sub-dictionary
names outkey1.
:returns: params: a validated and possibly filtered list of parameters.
"""
results = {}
if keepUnknownParams:
results = dict(params)
for entry in typeList:
key, dataType, outkey1, outkey2 = (list(entry) + [None] * 2)[:4]
if key in params:
if dataType == 'boolOrInt':
dataType = bool if str(params[key]).lower() in (
'true', 'false', 'on', 'off', 'yes', 'no') else int
try:
if dataType is bool:
results[key] = str(params[key]).lower() in (
'true', 'on', 'yes', '1')
else:
results[key] = dataType(params[key])
except ValueError:
raise RestException(
'"%s" parameter is an incorrect type.' % key)
if outkey1 is not None:
if outkey2 is not None:
results.setdefault(outkey1, {})[outkey2] = results[key]
else:
results[outkey1] = results[key]
del results[key]
return results
def _getTilesInfo(self, item, imageArgs):
"""
Get metadata for an item's large image.
:param item: the item to query.
:param imageArgs: additional arguments to use when fetching image data.
:return: the tile metadata.
"""
try:
return self.imageItemModel.getMetadata(item, **imageArgs)
except TileGeneralException as e:
raise RestException(e.args[0], code=400)
def _setContentDisposition(self, item, contentDisposition, mime, subname):
"""
If requested, set the content disposition and a suggested file name.
:param item: an item that includes a name.
:param contentDisposition: either 'inline' or 'attachemnt', otherwise
no header is added.
:param mime: the mimetype of the output image. Used for the filename
suffix.
:param subname: a subname to append to the item name.
"""
if (not item or not item.get('name') or
mime not in MimeTypeExtensions or
contentDisposition not in ('inline', 'attachment')):
return
filename = os.path.splitext(item['name'])[0]
if subname:
filename += '-' + subname
filename += '.' + MimeTypeExtensions[mime]
if not isinstance(filename, six.text_type):
filename = filename.decode('utf8', 'ignore')
safeFilename = filename.encode('ascii', 'ignore').replace(b'"', b'')
encodedFilename = six.moves.urllib.parse.quote(filename.encode('utf8', 'ignore'))
setResponseHeader(
'Content-Disposition',
'%s; filename="%s"; filename*=UTF-8\'\'%s' % (
contentDisposition, safeFilename, encodedFilename))
@describeRoute(
Description('Get large image metadata.')
.param('itemId', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getTilesInfo(self, item, params):
return self._getTilesInfo(item, params)
@describeRoute(
Description('Get large image internal metadata.')
.param('itemId', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getInternalMetadata(self, item, params):
try:
return self.imageItemModel.getInternalMetadata(item, **params)
except TileGeneralException as e:
raise RestException(e.args[0], code=400)
@describeRoute(
Description('Get test large image metadata.')
)
@access.public
def getTestTilesInfo(self, params):
item = {'largeImage': {'sourceName': 'test'}}
imageArgs = self._parseTestParams(params)
return self._getTilesInfo(item, imageArgs)
@describeRoute(
Description('Get DeepZoom compatible metadata.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('overlap', 'Pixel overlap (default 0), must be non-negative.',
required=False, dataType='int')
.param('tilesize', 'Tile size (default 256), must be a power of 2',
required=False, dataType='int')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getDZIInfo(self, item, params):
if 'encoding' in params and params['encoding'] not in ('JPEG', 'PNG'):
raise RestException('Only JPEG and PNG encodings are supported', code=400)
info = self._getTilesInfo(item, params)
tilesize = int(params.get('tilesize', 256))
if tilesize & (tilesize - 1):
raise RestException('Invalid tilesize', code=400)
overlap = int(params.get('overlap', 0))
if overlap < 0:
raise RestException('Invalid overlap', code=400)
result = ''.join([
'<?xml version="1.0" encoding="UTF-8"?>',
'<Image',
' TileSize="%d"' % tilesize,
' Overlap="%d"' % overlap,
' Format="%s"' % ('png' if params.get('encoding') == 'PNG' else 'jpg'),
' xmlns="http://schemas.microsoft.com/deepzoom/2008">',
'<Size',
' Width="%d"' % info['sizeX'],
' Height="%d"' % info['sizeY'],
'/>'
'</Image>',
])
setResponseHeader('Content-Type', 'text/xml')
setRawResponse()
return result
def _getTile(self, item, z, x, y, imageArgs, mayRedirect=False):
"""
Get an large image tile.
:param item: the item to get a tile from.
:param z: tile layer number (0 is the most zoomed-out).
.param x: the X coordinate of the tile (0 is the left side).
.param y: the Y coordinate of the tile (0 is the top).
:param imageArgs: additional arguments to use when fetching image data.
:param mayRedirect: if True or one of 'any', 'encoding', or 'exact',
allow return a response whcih may be a redirect.
:return: a function that returns the raw image data.
"""
try:
x, y, z = int(x), int(y), int(z)
except ValueError:
raise RestException('x, y, and z must be integers', code=400)
if x < 0 or y < 0 or z < 0:
raise RestException('x, y, and z must be positive integers',
code=400)
result = self.imageItemModel._tileFromHash(
item, x, y, z, mayRedirect=mayRedirect, **imageArgs)
if result is not None:
tileData, tileMime = result
else:
try:
tileData, tileMime = self.imageItemModel.getTile(
item, x, y, z, mayRedirect=mayRedirect, **imageArgs)
except TileGeneralException as e:
raise RestException(e.args[0], code=404)
setResponseHeader('Content-Type', tileMime)
setRawResponse()
return tileData
@describeRoute(
Description('Get a large image tile.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('z', 'The layer number of the tile (0 is the most zoomed-out '
'layer).', paramType='path')
.param('x', 'The X coordinate of the tile (0 is the left side).',
paramType='path')
.param('y', 'The Y coordinate of the tile (0 is the top).',
paramType='path')
.param('redirect', 'If the tile exists as a complete file, allow an '
'HTTP redirect instead of returning the data directly. The '
'redirect might not have the correct mime type. "exact" must '
'match the image encoding and quality parameters, "encoding" '
'must match the image encoding but disregards quality, and '
'"any" will redirect to any image if possible.', required=False,
enum=['false', 'exact', 'encoding', 'any'], default='false')
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
# Without caching, this checks for permissions every time. By using the
# LoadModelCache, three database lookups are avoided, which saves around
# 6 ms in tests. We also avoid the @access.public decorator and directly
# set the accessLevel attribute on the method.
# @access.public(cookie=True)
# @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
# def getTile(self, item, z, x, y, params):
# return self._getTile(item, z, x, y, params, True)
def getTile(self, itemId, z, x, y, params):
_adjustParams(params)
item = loadmodelcache.loadModel(
self, 'item', id=itemId, allowCookie=True, level=AccessType.READ)
_handleETag('getTile', item, z, x, y, params)
redirect = params.get('redirect', False)
if redirect not in ('any', 'exact', 'encoding'):
redirect = False
return self._getTile(item, z, x, y, params, mayRedirect=redirect)
getTile.accessLevel = 'public'
getTile.cookieAuth = True
@describeRoute(
Description('Get a large image tile with a frame number.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('frame', 'The frame number of the tile.', paramType='path')
.param('z', 'The layer number of the tile (0 is the most zoomed-out '
'layer).', paramType='path')
.param('x', 'The X coordinate of the tile (0 is the left side).',
paramType='path')
.param('y', 'The Y coordinate of the tile (0 is the top).',
paramType='path')
.param('redirect', 'If the tile exists as a complete file, allow an '
'HTTP redirect instead of returning the data directly. The '
'redirect might not have the correct mime type. "exact" must '
'match the image encoding and quality parameters, "encoding" '
'must match the image encoding but disregards quality, and '
'"any" will redirect to any image if possible.', required=False,
enum=['false', 'exact', 'encoding', 'any'], default='false')
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
# See getTile for caching rationale
def getTileWithFrame(self, itemId, frame, z, x, y, params):
_adjustParams(params)
item = loadmodelcache.loadModel(
self, 'item', id=itemId, allowCookie=True, level=AccessType.READ)
_handleETag('getTileWithFrame', item, frame, z, x, y, params)
redirect = params.get('redirect', False)
if redirect not in ('any', 'exact', 'encoding'):
redirect = False
params['frame'] = frame
return self._getTile(item, z, x, y, params, mayRedirect=redirect)
getTileWithFrame.accessLevel = 'public'
@describeRoute(
Description('Get a test large image tile.')
.param('z', 'The layer number of the tile (0 is the most zoomed-out '
'layer).', paramType='path')
.param('x', 'The X coordinate of the tile (0 is the left side).',
paramType='path')
.param('y', 'The Y coordinate of the tile (0 is the top).',
paramType='path')
.produces(ImageMimeTypes)
)
@access.public(cookie=True)
def getTestTile(self, z, x, y, params):
item = {'largeImage': {'sourceName': 'test'}}
imageArgs = self._parseTestParams(params)
return self._getTile(item, z, x, y, imageArgs)
@describeRoute(
Description('Get a DeepZoom image tile.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('level', 'The deepzoom layer number of the tile (8 is the '
'most zoomed-out layer).', paramType='path')
.param('xandy', 'The X and Y coordinate of the tile in the form '
'(x)_(y).(extension) where (0_0 is the left top).',
paramType='path')
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public(cookie=True)
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getDZITile(self, item, level, xandy, params):
_adjustParams(params)
tilesize = int(params.get('tilesize', 256))
if tilesize & (tilesize - 1):
raise RestException('Invalid tilesize', code=400)
overlap = int(params.get('overlap', 0))
if overlap < 0:
raise RestException('Invalid overlap', code=400)
x, y = [int(xy) for xy in xandy.split('.')[0].split('_')]
_handleETag('getDZITile', item, level, xandy, params)
metadata = self.imageItemModel.getMetadata(item, **params)
level = int(level)
maxlevel = int(math.ceil(math.log(max(
metadata['sizeX'], metadata['sizeY'])) / math.log(2)))
if level < 1 or level > maxlevel:
raise RestException('level must be between 1 and the image scale',
code=400)
lfactor = 2 ** (maxlevel - level)
region = {
'left': (x * tilesize - overlap) * lfactor,
'top': (y * tilesize - overlap) * lfactor,
'right': ((x + 1) * tilesize + overlap) * lfactor,
'bottom': ((y + 1) * tilesize + overlap) * lfactor,
}
width = height = tilesize + overlap * 2
if region['left'] < 0:
width += int(region['left'] / lfactor)
region['left'] = 0
if region['top'] < 0:
height += int(region['top'] / lfactor)
region['top'] = 0
if region['left'] >= metadata['sizeX']:
raise RestException('x is outside layer', code=400)
if region['top'] >= metadata['sizeY']:
raise RestException('y is outside layer', code=400)
if region['left'] < metadata['sizeX'] and region['right'] > metadata['sizeX']:
region['right'] = metadata['sizeX']
width = int(math.ceil(float(region['right'] - region['left']) / lfactor))
if region['top'] < metadata['sizeY'] and region['bottom'] > metadata['sizeY']:
region['bottom'] = metadata['sizeY']
height = int(math.ceil(float(region['bottom'] - region['top']) / lfactor))
regionData, regionMime = self.imageItemModel.getRegion(
item,
region=region,
output=dict(maxWidth=width, maxHeight=height),
**params)
setResponseHeader('Content-Type', regionMime)
setRawResponse()
return regionData
@describeRoute(
Description('Remove a large image from this item.')
.param('itemId', 'The ID of the item.', paramType='path')
)
@access.user
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
def deleteTiles(self, item, params):
deleted = self.imageItemModel.delete(item)
# TODO: a better response
return {
'deleted': deleted
}
@describeRoute(
Description('Get a thumbnail of a large image item.')
.notes('Aspect ratio is always preserved. If both width and height '
'are specified, the resulting thumbnail may be smaller in one '
'of the two dimensions. If neither width nor height is given, '
'a default size will be returned. '
'This creates a thumbnail from the lowest level of the source '
'image, which means that asking for a large thumbnail will not '
'be a high-quality image.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('width', 'The maximum width of the thumbnail in pixels.',
required=False, dataType='int')
.param('height', 'The maximum height of the thumbnail in pixels.',
required=False, dataType='int')
.param('fill', 'A fill color. If width and height are both specified '
'and fill is specified and not "none", the output image is '
'padded on either the sides or the top and bottom to the '
'requested output size. Most css colors are accepted.',
required=False)
.param('frame', 'For multiframe images, the 0-based frame number. '
'This is ignored on non-multiframe images.', required=False,
dataType='int')
.param('encoding', 'Thumbnail output encoding', required=False,
enum=['JPEG', 'PNG', 'TIFF'], default='JPEG')
.param('contentDisposition', 'Specify the Content-Disposition response '
'header disposition-type value.', required=False,
enum=['inline', 'attachment'])
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public(cookie=True)
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getTilesThumbnail(self, item, params):
_adjustParams(params)
params = self._parseParams(params, True, [
('width', int),
('height', int),
('fill', str),
('frame', int),
('jpegQuality', int),
('jpegSubsampling', int),
('tiffCompression', str),
('encoding', str),
('style', str),
('contentDisposition', str),
])
_handleETag('getTilesThumbnail', item, params)
try:
result = self.imageItemModel.getThumbnail(item, **params)
except TileGeneralException as e:
raise RestException(e.args[0])
except ValueError as e:
raise RestException('Value Error: %s' % e.args[0])
if not isinstance(result, tuple):
return result
thumbData, thumbMime = result
self._setContentDisposition(
item, params.get('contentDisposition'), thumbMime, 'thumbnail')
setResponseHeader('Content-Type', thumbMime)
setRawResponse()
return thumbData
@describeRoute(
Description('Get any region of a large image item, optionally scaling '
'it.')
.notes('If neither width nor height is specified, the full resolution '
'region is returned. If a width or height is specified, '
'aspect ratio is always preserved (if both are given, the '
'resulting image may be smaller in one of the two '
'dimensions). When scaling must be applied, the image is '
'downsampled from a higher resolution layer, never upsampled.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('left', 'The left column (0-based) of the region to process. '
'Negative values are offsets from the right edge.',
required=False, dataType='float')
.param('top', 'The top row (0-based) of the region to process. '
'Negative values are offsets from the bottom edge.',
required=False, dataType='float')
.param('right', 'The right column (0-based from the left) of the '
'region to process. The region will not include this column. '
'Negative values are offsets from the right edge.',
required=False, dataType='float')
.param('bottom', 'The bottom row (0-based from the top) of the region '
'to process. The region will not include this row. Negative '
'values are offsets from the bottom edge.',
required=False, dataType='float')
.param('regionWidth', 'The width of the region to process.',
required=False, dataType='float')
.param('regionHeight', 'The height of the region to process.',
required=False, dataType='float')
.param('units', 'Units used for left, top, right, bottom, '
'regionWidth, and regionHeight. base_pixels are pixels at the '
'maximum resolution, pixels and mm are at the specified '
'magnfication, fraction is a scale of [0-1].', required=False,
enum=sorted(set(TileInputUnits.values())),
default='base_pixels')
.param('width', 'The maximum width of the output image in pixels.',
required=False, dataType='int')
.param('height', 'The maximum height of the output image in pixels.',
required=False, dataType='int')
.param('fill', 'A fill color. If output dimensions are specified and '
'fill is specified and not "none", the output image is padded '
'on either the sides or the top and bottom to the requested '
'output size. Most css colors are accepted.', required=False)
.param('magnification', 'Magnification of the output image. If '
'neither width for height is specified, the magnification, '
'mm_x, and mm_y parameters are used to select the output size.',
required=False, dataType='float')
.param('mm_x', 'The size of the output pixels in millimeters',
required=False, dataType='float')
.param('mm_y', 'The size of the output pixels in millimeters',
required=False, dataType='float')
.param('exact', 'If magnification, mm_x, or mm_y are specified, they '
'must match an existing level of the image exactly.',
required=False, dataType='boolean', default=False)
.param('frame', 'For multiframe images, the 0-based frame number. '
'This is ignored on non-multiframe images.', required=False,
dataType='int')
.param('encoding', 'Output image encoding', required=False,
enum=['JPEG', 'PNG', 'TIFF'], default='JPEG')
.param('jpegQuality', 'Quality used for generating JPEG images',
required=False, dataType='int', default=95)
.param('jpegSubsampling', 'Chroma subsampling used for generating '
'JPEG images. 0, 1, and 2 are full, half, and quarter '
'resolution chroma respectively.', required=False,
enum=['0', '1', '2'], dataType='int', default='0')
.param('tiffCompression', 'Compression method when storing a TIFF '
'image', required=False,
enum=['raw', 'tiff_lzw', 'jpeg', 'tiff_adobe_deflate'])
.param('style', 'JSON-encoded style string', required=False)
.param('resample', 'If false, an existing level of the image is used '
'for the histogram. If true, the internal values are '
'interpolated to match the specified size as needed. 0-3 for '
'a specific interpolation method (0-nearest, 1-lanczos, '
'2-bilinear, 3-bicubic)', required=False,
enum=['false', 'true', '0', '1', '2', '3'], default='false')
.param('contentDisposition', 'Specify the Content-Disposition response '
'header disposition-type value.', required=False,
enum=['inline', 'attachment'])
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
.errorResponse('Insufficient memory.')
)
@access.public(cookie=True)
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getTilesRegion(self, item, params):
_adjustParams(params)
params = self._parseParams(params, True, [
('left', float, 'region', 'left'),
('top', float, 'region', 'top'),
('right', float, 'region', 'right'),
('bottom', float, 'region', 'bottom'),
('regionWidth', float, 'region', 'width'),
('regionHeight', float, 'region', 'height'),
('units', str, 'region', 'units'),
('unitsWH', str, 'region', 'unitsWH'),
('width', int, 'output', 'maxWidth'),
('height', int, 'output', 'maxHeight'),
('fill', str),
('magnification', float, 'scale', 'magnification'),
('mm_x', float, 'scale', 'mm_x'),
('mm_y', float, 'scale', 'mm_y'),
('exact', bool, 'scale', 'exact'),
('frame', int),
('encoding', str),
('jpegQuality', int),
('jpegSubsampling', int),
('tiffCompression', str),
('style', str),
('resample', 'boolOrInt'),
('contentDisposition', str),
])
_handleETag('getTilesRegion', item, params)
try:
regionData, regionMime = self.imageItemModel.getRegion(
item, **params)
except TileGeneralException as e:
raise RestException(e.args[0])
except ValueError as e:
raise RestException('Value Error: %s' % e.args[0])
self._setContentDisposition(
item, params.get('contentDisposition'), regionMime, 'region')
setResponseHeader('Content-Type', regionMime)
setRawResponse()
return regionData
@describeRoute(
Description('Get a single pixel of a large image item.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('left', 'The left column (0-based) of the pixel.',
required=False, dataType='float')
.param('top', 'The top row (0-based) of the pixel.',
required=False, dataType='float')
.param('units', 'Units used for left and top. base_pixels are pixels '
'at the maximum resolution, pixels and mm are at the specified '
'magnfication, fraction is a scale of [0-1].', required=False,
enum=sorted(set(TileInputUnits.values())),
default='base_pixels')
.param('frame', 'For multiframe images, the 0-based frame number. '
'This is ignored on non-multiframe images.', required=False,
dataType='int')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public(cookie=True)
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getTilesPixel(self, item, params):
params = self._parseParams(params, True, [
('left', float, 'region', 'left'),
('top', float, 'region', 'top'),
('right', float, 'region', 'right'),
('bottom', float, 'region', 'bottom'),
('units', str, 'region', 'units'),
('frame', int),
])
try:
pixel = self.imageItemModel.getPixel(item, **params)
except TileGeneralException as e:
raise RestException(e.args[0])
except ValueError as e:
raise RestException('Value Error: %s' % e.args[0])
return pixel
@describeRoute(
Description('Get a histogram for any region of a large image item.')
.notes('This can take all of the parameters as the region endpoint, '
'plus some histogram-specific parameters. Only typically used '
'parameters are listed. The returned result is a list with '
'one entry per channel (always one of L, LA, RGB, or RGBA '
'colorspace). Each entry has the histogram values, bin edges, '
'minimum and maximum values for the channel, and number of '
'samples (pixels) used in the computation.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('width', 'The maximum width of the analyzed region in pixels.',
default=2048, required=False, dataType='int')
.param('height', 'The maximum height of the analyzed region in pixels.',
default=2048, required=False, dataType='int')
.param('resample', 'If false, an existing level of the image is used '
'for the histogram. If true, the internal values are '
'interpolated to match the specified size as needed. 0-3 for '
'a specific interpolation method (0-nearest, 1-lanczos, '
'2-bilinear, 3-bicubic)', required=False,
enum=['false', 'true', '0', '1', '2', '3'], default='false')
.param('frame', 'For multiframe images, the 0-based frame number. '
'This is ignored on non-multiframe images.', required=False,
dataType='int')
.param('bins', 'The number of bins in the histogram.',
default=256, required=False, dataType='int')
.param('rangeMin', 'The minimum value in the histogram. Defaults to '
'the minimum value in the image.',
required=False, dataType='float')
.param('rangeMax', 'The maximum value in the histogram. Defaults to '
'the maximum value in the image.',
required=False, dataType='float')
.param('density', 'If true, scale the results by the number of '
'samples.', required=False, dataType='boolean', default=False)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getHistogram(self, item, params):
_adjustParams(params)
params = self._parseParams(params, True, [
('left', float, 'region', 'left'),
('top', float, 'region', 'top'),
('right', float, 'region', 'right'),
('bottom', float, 'region', 'bottom'),
('regionWidth', float, 'region', 'width'),
('regionHeight', float, 'region', 'height'),
('units', str, 'region', 'units'),
('unitsWH', str, 'region', 'unitsWH'),
('width', int, 'output', 'maxWidth'),
('height', int, 'output', 'maxHeight'),
('fill', str),
('magnification', float, 'scale', 'magnification'),
('mm_x', float, 'scale', 'mm_x'),
('mm_y', float, 'scale', 'mm_y'),
('exact', bool, 'scale', 'exact'),
('frame', int),
('encoding', str),
('jpegQuality', int),
('jpegSubsampling', int),
('tiffCompression', str),
('style', str),
('resample', 'boolOrInt'),
('bins', int),
('rangeMin', int),
('rangeMax', int),
('density', bool),
])
_handleETag('getHistogram', item, params)
histRange = None
if 'rangeMin' in params or 'rangeMax' in params:
histRange = [params.pop('rangeMin', 0), params.pop('rangeMax', 256)]
result = self.imageItemModel.histogram(item, range=histRange, **params)
result = result['histogram']
# Cast everything to lists and floats so json with encode properly
for entry in result:
for key in {'bin_edges', 'hist', 'range'}:
if key in entry:
entry[key] = [float(val) for val in list(entry[key])]
for key in {'min', 'max', 'samples'}:
if key in entry:
entry[key] = float(entry[key])
return result
@describeRoute(
Description('Get a list of additional images associated with a large image.')
.param('itemId', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public
@loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.READ)
def getAssociatedImagesList(self, item, params):
try:
return self.imageItemModel.getAssociatedImagesList(item)
except TileGeneralException as e:
raise RestException(e.args[0], code=400)
@describeRoute(
Description('Get an image associated with a large image.')
.notes('Because associated images may contain PHI, admin access to '
'the item is required.')
.param('itemId', 'The ID of the item.', paramType='path')
.param('image', 'The key of the associated image.', paramType='path')
.param('width', 'The maximum width of the image in pixels.',
required=False, dataType='int')
.param('height', 'The maximum height of the image in pixels.',
required=False, dataType='int')
.param('encoding', 'Image output encoding', required=False,
enum=['JPEG', 'PNG', 'TIFF'], default='JPEG')
.param('contentDisposition', 'Specify the Content-Disposition response '
'header disposition-type value.', required=False,
enum=['inline', 'attachment'])
.produces(ImageMimeTypes)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
@access.public(cookie=True)
def getAssociatedImage(self, itemId, image, params):
_adjustParams(params)
# We can't use the loadmodel decorator, as we want to allow cookies
item = loadmodelcache.loadModel(
self, 'item', id=itemId, allowCookie=True, level=AccessType.READ)
params = self._parseParams(params, True, [
('width', int),
('height', int),
('jpegQuality', int),
('jpegSubsampling', int),
('tiffCompression', str),
('encoding', str),
('style', str),
('contentDisposition', str),
])
_handleETag('getAssociatedImage', item, image, params)
try:
result = self.imageItemModel.getAssociatedImage(item, image, **params)
except TileGeneralException as e:
raise RestException(e.args[0], code=400)
if not isinstance(result, tuple):
return result
imageData, imageMime = result
self._setContentDisposition(
item, params.get('contentDisposition'), imageMime, image)
setResponseHeader('Content-Type', imageMime)
setRawResponse()
return imageData
conn.unbind_s()
user = _getLdapUser(attrs, server)
if user:
event.stopPropagation().preventDefault().addResponse(user)
except ldap.LDAPError:
logger.exception('LDAP connection exception (%s).' % server['uri'])
continue
@access.admin
@boundHandler
@autoDescribeRoute(
Description('Test connection status to a LDAP server.').notes(
'You must be an administrator to call this.').param(
'uri', 'The URI of the server.').param(
'bindName', 'The LDAP identity to bind with.').param(
'password', 'Password to bind with.').errorResponse(
'You are not an administrator.', 403))
def _ldapServerTest(self, uri, bindName, password, params):
conn = None
try:
conn = ldap.initialize(uri)
conn.set_option(ldap.OPT_TIMEOUT, _CONNECT_TIMEOUT)
conn.set_option(ldap.OPT_NETWORK_TIMEOUT, _CONNECT_TIMEOUT)
conn.bind_s(bindName, password, ldap.AUTH_SIMPLE)
return {'connected': True}
except ldap.LDAPError as e:
return {
'connected':
False,
'error':
def genHandlerToRunDockerCLI(dockerImage, cliRelPath, cliXML, restResource):
"""Generates a handler to run docker CLI using girder_worker
Parameters
----------
dockerImage : str
Docker image in which the CLI resides
cliRelPath : str
Relative path of the CLI which is needed to run the CLI by running
the command docker run `dockerImage` `cliRelPath`
cliXML:str
Cached copy of xml spec for this cli
restResource : girder.api.rest.Resource
The object of a class derived from girder.api.rest.Resource to which
this handler will be attached
Returns
-------
function
Returns a function that runs the CLI using girder_worker
"""
cliName = os.path.normpath(cliRelPath).replace(os.sep, '.')
# get xml spec
str_xml = cliXML
# parse cli xml spec
with tempfile.NamedTemporaryFile(suffix='.xml') as f:
f.write(str_xml)
f.flush()
clim = CLIModule(f.name)
# create CLI description string
str_description = ['Description: <br/><br/>' + clim.description]
if clim.version is not None and len(clim.version) > 0:
str_description.append('Version: ' + clim.version)
if clim.license is not None and len(clim.license) > 0:
str_description.append('License: ' + clim.license)
if clim.contributor is not None and len(clim.contributor) > 0:
str_description.append('Author(s): ' + clim.contributor)
if clim.acknowledgements is not None and \
len(clim.acknowledgements) > 0:
str_description.append('Acknowledgements: ' + clim.acknowledgements)
str_description = '<br/><br/>'.join(str_description)
# do stuff needed to create REST endpoint for cLI
handlerDesc = Description(clim.title).notes(str_description)
# get CLI parameters
index_params, opt_params, simple_out_params = _getCLIParameters(clim)
# print index_params [<CLIParameter 'inputMultipleImage' of type directory>, <CLIParameter 'outputThresholding' of type file>, <CLIParameter 'tableFile' of type file>]
# add indexed input parameters
index_input_params = filter(lambda p: p.channel != 'output', index_params)
# print index_input_params [<CLIParameter 'inputMultipleImage' of type directory>]
_addIndexedInputParamsToHandler(index_input_params, handlerDesc)
# add indexed output parameters
index_output_params = filter(lambda p: p.channel == 'output', index_params)
_addIndexedOutputParamsToHandler(index_output_params, handlerDesc)
# add optional input parameters
opt_input_params = filter(lambda p: p.channel != 'output', opt_params)
_addOptionalInputParamsToHandler(opt_input_params, handlerDesc)
# add optional output parameters
opt_output_params = filter(lambda p: p.channel == 'output', opt_params)
_addOptionalOutputParamsToHandler(opt_output_params, handlerDesc)
# add returnparameterfile if there are simple output params
if len(simple_out_params) > 0:
_addReturnParameterFileParamToHandler(handlerDesc)
# define CLI handler function
@boundHandler(restResource)
@access.user
@describeRoute(handlerDesc)
def cliHandler(self, **hargs):
print 'in cliHandler hargs is '
print hargs
user = self.getCurrentUser()
token = self.getCurrentToken()['_id']
# create job
jobModel = self.model('job', 'jobs')
jobTitle = '.'.join((restResource.resourceName, cliName))
job = jobModel.createJob(title=jobTitle,
type=jobTitle,
handler='worker_handler',
user=user)
kwargs = {
'validate': False,
'auto_convert': True,
'cleanup': True,
'inputs': dict(),
'outputs': dict()
}
# create job info
jobToken = jobModel.createJobToken(job)
kwargs['jobInfo'] = wutils.jobInfoSpec(job, jobToken)
# initialize task spec
taskSpec = {
'name': cliName,
'mode': 'docker',
'docker_image': dockerImage,
'pull_image': False,
'inputs': [],
'outputs': []
}
_addIndexedInputParamsToTaskSpec(index_input_params, taskSpec)
_addIndexedOutputParamsToTaskSpec(index_output_params, taskSpec, hargs)
_addOptionalInputParamsToTaskSpec(opt_input_params, taskSpec)
_addOptionalOutputParamsToTaskSpec(opt_output_params, taskSpec, hargs)
if len(simple_out_params) > 0:
_addReturnParameterFileParamToTaskSpec(taskSpec, hargs)
kwargs['task'] = taskSpec
# add input/output parameter bindings
_addIndexedInputParamBindings(index_input_params, kwargs['inputs'],
hargs, token)
_addIndexedOutputParamBindings(index_output_params, kwargs['outputs'],
hargs, user, token)
_addOptionalInputParamBindings(opt_input_params, kwargs['inputs'],
hargs, user, token)
_addOptionalOutputParamBindings(opt_output_params, kwargs['outputs'],
hargs, user, token)
if len(simple_out_params) > 0:
_addReturnParameterFileBinding(kwargs['outputs'], hargs, user,
token)
# construct container arguments
containerArgs = [cliRelPath]
_addOptionalInputParamsToContainerArgs(opt_input_params, containerArgs,
hargs)
_addOptionalOutputParamsToContainerArgs(opt_output_params,
containerArgs, kwargs, hargs)
_addReturnParameterFileToContainerArgs(containerArgs, kwargs, hargs)
_addIndexedParamsToContainerArgs(index_params, containerArgs, hargs)
taskSpec['container_args'] = containerArgs
# schedule job
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
# return result
return jobModel.filter(job, user)
handlerFunc = cliHandler
# loadmodel stuff for indexed input params on girder
index_input_params_on_girder = filter(_is_on_girder, index_input_params)
for param in index_input_params_on_girder:
curModel = _SLICER_TYPE_TO_GIRDER_MODEL_MAP[param.typ]
if curModel != 'url':
suffix = _SLICER_TYPE_TO_GIRDER_INPUT_SUFFIX_MAP[param.typ]
curMap = {param.identifier() + suffix: param.identifier()}
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.READ)(handlerFunc)
# loadmodel stuff for indexed output params on girder
index_output_params_on_girder = filter(_is_on_girder, index_output_params)
for param in index_output_params_on_girder:
curModel = 'folder'
curMap = {
param.identifier() + _girderOutputFolderSuffix: param.identifier()
}
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.WRITE)(handlerFunc)
return handlerFunc
class QuotaPolicy(Resource):
def _filter(self, model, resource):
"""
Filter a resource to include only the ordinary data and the quota
field.
:param model: the type of resource (e.g., user or collection)
:param resource: the resource document.
:returns: filtered field of the resource with the quota data, if any.
"""
filtered = self.model(model).filter(resource, self.getCurrentUser())
filtered[QUOTA_FIELD] = resource.get(QUOTA_FIELD, {})
return filtered
def _setResourceQuota(self, model, resource, policy):
"""
Handle setting quota policies for any resource that supports them.
:param model: the type of resource (e.g., user or collection)
:param resource: the resource document.
:param params: the query parameters. 'policy' is required and used.
:returns: the updated resource document.
"""
policy = self._validatePolicy(policy)
if QUOTA_FIELD not in resource:
resource[QUOTA_FIELD] = {}
resource[QUOTA_FIELD].update(policy)
self.model(model).save(resource, validate=False)
return self._filter(model, resource)
def _validate_fallbackAssetstore(self, value):
"""Validate the fallbackAssetstore parameter.
:param value: the proposed value.
:returns: the validated value: either None or 'current' to use the
current assetstore, 'none' to disable a fallback assetstore,
or an assetstore ID.
"""
if not value or value == 'current':
return None
if value == 'none':
return value
try:
value = ObjectId(value)
except InvalidId:
raise RestException(
'Invalid fallbackAssetstore. Must either be an assetstore '
'ID, be blank or "current" to use the current assetstore, or '
'be "none" to disable fallback usage.',
extra='fallbackAssetstore')
return value
def _validate_fileSizeQuota(self, value):
"""Validate the fileSizeQuota parameter.
:param value: the proposed value.
:returns: the validated value
:rtype: None or int
"""
(value, err) = ValidateSizeQuota(value)
if err:
raise RestException(err, extra='fileSizeQuota')
return value
def _validate_preferredAssetstore(self, value):
"""Validate the preferredAssetstore parameter.
:param value: the proposed value.
:returns: the validated value: either None or 'current' to use the
current assetstore or an assetstore ID.
"""
if not value or value == 'current':
return None
try:
value = ObjectId(value)
except InvalidId:
raise RestException(
'Invalid preferredAssetstore. Must either be an assetstore '
'ID, or be blank or "current" to use the current assetstore.',
extra='preferredAssetstore')
return value
def _validate_useQuotaDefault(self, value):
"""Validate the useQuotaDefault parameter.
:param value: the proposed value.
:returns: the validated value
:rtype: None or bool
"""
if str(value).lower() in ('none', 'true', 'yes', '1'):
return True
if str(value).lower() in ('false', 'no', '0'):
return False
raise RestException(
'Invalid useQuotaDefault. Must either be true or false.',
extra='useQuotaDefault')
def _validatePolicy(self, policy):
"""
Validate a policy JSON object. Only a limited set of keys is
supported, and each of them has a restricted data type.
:param policy: JSON object to validate. This may also be a Python
dictionary as if the JSON was already decoded.
:returns: a validate policy dictionary.
"""
validKeys = []
for key in dir(self):
if key.startswith('_validate_'):
validKeys.append(key.split('_validate_', 1)[1])
for key in list(policy):
if key.startswith('_'):
del policy[key]
for key in policy:
if key not in validKeys:
raise RestException(
'%s is not a valid quota policy key. Valid keys are %s.' %
(key, ', '.join(sorted(validKeys))))
funcName = '_validate_' + key
policy[key] = getattr(self, funcName)(policy[key])
return policy
@access.public
@autoDescribeRoute(
Description('Get quota and assetstore policies for the collection.').
modelParam('id',
'The collection ID',
model=Collection,
level=AccessType.READ).errorResponse(
'ID was invalid.').errorResponse(
'Read permission denied on the collection.', 403))
def getCollectionQuota(self, collection):
if QUOTA_FIELD not in collection:
collection[QUOTA_FIELD] = {}
collection[QUOTA_FIELD][
'_currentFileSizeQuota'] = self._getFileSizeQuota(
'collection', collection)
return self._filter('collection', collection)
@access.public
@autoDescribeRoute(
Description('Set quota and assetstore policies for the collection.').
modelParam(
'id',
'The collection ID',
model=Collection,
level=AccessType.ADMIN).jsonParam(
'policy', 'A JSON object containing the policies. This is a '
'dictionary of keys and values. Any key that is not specified '
'does not change.',
requireObject=True).errorResponse(
'ID was invalid.').errorResponse(
'Read permission denied on the collection.', 403))
def setCollectionQuota(self, collection, policy):
return self._setResourceQuota('collection', collection, policy)
@access.public
@autoDescribeRoute(
Description(
'Get quota and assetstore policies for the user.').modelParam(
'id', 'The user ID', model=User,
level=AccessType.READ).errorResponse(
'ID was invalid.').errorResponse(
'Read permission denied on the user.', 403))
def getUserQuota(self, user):
if QUOTA_FIELD not in user:
user[QUOTA_FIELD] = {}
user[QUOTA_FIELD]['_currentFileSizeQuota'] = self._getFileSizeQuota(
'user', user)
return self._filter('user', user)
@access.public
@autoDescribeRoute(
Description(
'Set quota and assetstore policies for the user.').modelParam(
'id', 'The user ID', model=User, level=AccessType.ADMIN).
jsonParam(
'policy', 'A JSON object containing the policies. This is a '
'dictionary of keys and values. Any key that is not specified '
'does not change.',
requireObject=True).errorResponse('ID was invalid.').errorResponse(
'Read permission denied on the user.', 403))
def setUserQuota(self, user, policy):
return self._setResourceQuota('user', user, policy)
def _checkAssetstore(self, assetstoreSpec):
"""
Check is a specified assetstore is available.
:param assetstoreSpec: None for use current assetstore, 'none' to
disallow the assetstore, or an assetstore ID to
check if that assetstore exists and is nominally
available.
:returns: None to use the current assetstore, False to indicate no
assetstore is allowed, or an assetstore document of an
allowed assetstore.
"""
if assetstoreSpec is None:
return None
if assetstoreSpec == 'none':
return False
assetstore = Assetstore().load(id=assetstoreSpec)
if not assetstore:
return False
adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
if getattr(adapter, 'unavailable', False):
return False
return assetstore
def _getBaseResource(self, model, resource):
"""
Get the base resource for something pertaining to quota policies. If
the base resource has no quota policy, return (None, None).
:param model: the initial model type. Could be file, item, folder,
user, or collection.
:param resource: the initial resource document.
:returns: A pair ('model', 'resource'), where 'model' is the base model
type, either 'user' or 'collection'., and 'resource' is the
base resource document or the id of that document.
"""
if isinstance(resource, tuple(list(six.string_types) + [ObjectId])):
resource = self.model(model).load(id=resource, force=True)
if model == 'file':
model = 'item'
resource = Item().load(id=resource['itemId'], force=True)
if model in ('folder', 'item'):
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
resource = self.model(model).load(id=resource['_id'],
force=True)
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
return None, None
model = resource['baseParentType']
resourceId = resource['baseParentId']
resource = self.model(model).load(id=resourceId, force=True)
if model in ('user', 'collection') and resource:
# Ensure the base resource has a quota field so we can use the
# default quota if appropriate
if QUOTA_FIELD not in resource:
resource[QUOTA_FIELD] = {}
if not resource or QUOTA_FIELD not in resource:
return None, None
return model, resource
def getUploadAssetstore(self, event):
"""
Handle the model.upload.assetstore event. This event passes a
dictionary consisting of a model type and resource document. If the
base document has an assetstore policy, then set an assetstore key of
this dictionary to an assetstore document that should be used or
prevent the default action if no appropriate assetstores are allowed.
:param event: event record.
"""
model, resource = self._getBaseResource(event.info['model'],
event.info['resource'])
if resource is None:
return
policy = resource[QUOTA_FIELD]
assetstore = self._checkAssetstore(
policy.get('preferredAssetstore', None))
if assetstore is False:
assetstore = self._checkAssetstore(
policy.get('fallbackAssetstore', None))
if assetstore is not False:
logger.info(
'preferredAssetstore not available for %s %s, '
'using fallbackAssetstore', model, resource['_id'])
if assetstore is False:
raise GirderException('Required assetstore is unavailable')
if assetstore:
event.addResponse(assetstore)
def _getFileSizeQuota(self, model, resource):
"""
Get the current fileSizeQuota for a resource. This takes the default
quota into account if necessary.
:param model: the type of resource (e.g., user or collection)
:param resource: the resource document.
:returns: the fileSizeQuota. None for no quota (unlimited), otherwise
a positive integer.
"""
useDefault = resource[QUOTA_FIELD].get('useQuotaDefault', True)
quota = resource[QUOTA_FIELD].get('fileSizeQuota', None)
if useDefault:
if model == 'user':
key = constants.PluginSettings.QUOTA_DEFAULT_USER_QUOTA
elif model == 'collection':
key = constants.PluginSettings.QUOTA_DEFAULT_COLLECTION_QUOTA
else:
key = None
if key:
quota = Setting().get(key, None)
if not quota or quota < 0 or not isinstance(quota, six.integer_types):
return None
return quota
def _checkUploadSize(self, upload):
"""
Check if an upload will fit within a quota restriction.
:param upload: an upload document.
:returns: None if the upload is allowed, otherwise a dictionary of
information about the quota restriction.
"""
origSize = 0
if 'fileId' in upload:
file = File().load(id=upload['fileId'], force=True)
origSize = int(file.get('size', 0))
model, resource = self._getBaseResource('file', file)
else:
model, resource = self._getBaseResource(upload['parentType'],
upload['parentId'])
if resource is None:
return None
fileSizeQuota = self._getFileSizeQuota(model, resource)
if not fileSizeQuota:
return None
newSize = resource['size'] + upload['size'] - origSize
# always allow replacement with a smaller object
if newSize <= fileSizeQuota or upload['size'] < origSize:
return None
left = fileSizeQuota - resource['size']
if left < 0:
left = 0
return {
'fileSizeQuota': fileSizeQuota,
'sizeNeeded': upload['size'] - origSize,
'quotaLeft': left,
'quotaUsed': resource['size']
}
def checkUploadStart(self, event):
"""
Check if an upload will fit within a quota restriction. This is before
the upload occurs, but since multiple uploads can be started
concurrently, we also have to check when the upload is being completed.
:param event: event record.
"""
if '_id' in event.info:
return
quotaInfo = self._checkUploadSize(event.info)
if quotaInfo:
raise ValidationException(
'Upload would exceed file storage quota (need %s, only %s '
'available - used %s out of %s)' %
(formatSize(quotaInfo['sizeNeeded']),
formatSize(quotaInfo['quotaLeft']),
formatSize(quotaInfo['quotaUsed']),
formatSize(quotaInfo['fileSizeQuota'])),
field='size')
def checkUploadFinalize(self, event):
"""
Check if an upload will fit within a quota restriction before
finalizing it. If it doesn't, discard it.
:param event: event record.
"""
upload = event.info
quotaInfo = self._checkUploadSize(upload)
if quotaInfo:
# Delete the upload
Upload().cancelUpload(upload)
raise ValidationException(
'Upload exceeded file storage quota (need %s, only %s '
'available - used %s out of %s)' %
(formatSize(quotaInfo['sizeNeeded']),
formatSize(quotaInfo['quotaLeft']),
formatSize(quotaInfo['quotaUsed']),
formatSize(quotaInfo['fileSizeQuota'])),
field='size')
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
from girder.api import access
from girder.api.describe import Description, autoDescribeRoute
from girder.api.rest import boundHandler
from girder.constants import SettingDefault
from .constants import PluginSettings
@access.user
@boundHandler
@autoDescribeRoute(
Description('Get list of item licenses.').param(
'default',
'Whether to return the default list of item licenses.',
required=False,
dataType='boolean',
default=False))
def getLicenses(self, default):
if default:
licenses = SettingDefault.defaults[PluginSettings.LICENSES]
else:
licenses = self.model('setting').get(PluginSettings.LICENSES)
return licenses
class Viame(Resource):
def __init__(self, pipelines=[]):
super(Viame, self).__init__()
self.resourceName = "viame"
self.pipelines = pipelines
self.route("GET", ("pipelines", ), self.get_pipelines)
self.route("POST", ("pipeline", ), self.run_pipeline_task)
self.route("POST", ("postprocess", ":id"), self.postprocess)
self.route("POST", ("attribute", ), self.create_attribute)
self.route("GET", ("attribute", ), self.get_attributes)
self.route("PUT", ("attribute", ":id"), self.update_attribute)
self.route("POST", ("validate_files", ), self.validate_files)
self.route("DELETE", ("attribute", ":id"), self.delete_attribute)
self.route("GET", ("valid_images", ), self.get_valid_images)
@access.user
@describeRoute(Description("Get available pipelines"))
def get_pipelines(self, params):
return self.pipelines
@access.user
@autoDescribeRoute(
Description("Run viame pipeline").modelParam(
"folderId",
description="Folder id of a video clip",
model=Folder,
paramType="query",
required=True,
level=AccessType.READ,
).param(
"pipeline",
"Pipeline to run against the video",
default="detector_simple_hough.pipe",
))
def run_pipeline_task(self, folder, pipeline):
user = self.getCurrentUser()
token = Token().createToken(user=user, days=1)
move_existing_result_to_auxiliary_folder(folder, user)
input_type = folder["meta"]["type"]
return run_pipeline.delay(
GetPathFromFolderId(str(folder["_id"])),
str(folder["_id"]),
pipeline,
input_type,
girder_job_title=("Running {} on {}".format(
pipeline, str(folder["name"]))),
girder_client_token=str(token["_id"]),
)
@access.user
@autoDescribeRoute(
Description(
"Test whether or not a set of files are safe to upload").jsonParam(
"files", "", paramType="body"))
def validate_files(self, files):
ok = True
message = ""
mediatype = ""
videos = [f for f in files if videoRegex.search(f)]
csvs = [f for f in files if csvRegex.search(f)]
images = [f for f in files if imageRegex.search(f)]
ymls = [f for f in files if ymlRegex.search(f)]
if len(videos) and len(images):
ok = False
message = "Do not upload images and videos in the same batch."
elif len(csvs) > 1:
ok = False
message = "Can only upload a single CSV Annotation per import"
elif len(csvs) == 1 and len(ymls):
ok = False
message = "Cannot mix annotation import types"
elif len(videos) > 1 and (len(csvs) or len(ymls)):
ok = False
message = (
"Annotation upload is not supported when multiple videos are uploaded"
)
elif (not len(videos)) and (not len(images)):
ok = False
message = "No supported media-type files found"
elif len(videos):
mediatype = 'video'
elif len(images):
mediatype = 'image-sequence'
return {
"ok": ok,
"message": message,
"type": mediatype,
"media": images + videos,
"annotations": csvs + ymls,
}
@access.user
@autoDescribeRoute(
Description("Post-processing to be run after media/annotation import").
modelParam(
"id",
description="Folder containing the items to process",
model=Folder,
level=AccessType.WRITE,
).param(
"skipJobs",
"Whether to skip processing that might dispatch worker jobs",
paramType="formData",
dataType="boolean",
default=False,
required=False,
))
def postprocess(self, folder, skipJobs):
"""
Post-processing to be run after media/annotation import
When skipJobs=False, the following may run as jobs:
Transcoding of Video
Transcoding of Images
Conversion of KPF annotations into track JSON
In either case, the following may run synchronously:
Conversion of CSV annotations into track JSON
"""
user = self.getCurrentUser()
auxiliary = get_or_create_auxiliary_folder(folder, user)
if not skipJobs:
token = Token().createToken(user=user, days=1)
# transcode VIDEO if necessary
videoItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": videoRegex
}})
for item in videoItems:
convert_video.delay(
GetPathFromItemId(str(item["_id"])),
str(item["folderId"]),
auxiliary["_id"],
girder_job_title=(
"Converting {} to a web friendly format".format(
str(item["_id"]))),
girder_client_token=str(token["_id"]),
)
# transcode IMAGERY if necessary
imageItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": imageRegex
}})
safeImageItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": safeImageRegex
}})
if imageItems.count() > safeImageItems.count():
convert_images.delay(
folder["_id"],
girder_client_token=str(token["_id"]),
girder_job_title=
f"Converting {folder['_id']} to a web friendly format",
)
elif imageItems.count() > 0:
folder["meta"]["annotate"] = True
# transform KPF if necessary
ymlItems = Folder().childItems(
folder, filters={"lowerName": {
"$regex": ymlRegex
}})
if ymlItems.count() > 0:
# There might be up to 3 yamls
allFiles = [Item().childFiles(item)[0] for item in ymlItems]
saveTracks(folder,
meva_serializer.load_kpf_as_tracks(allFiles), user)
ymlItems.rewind()
for item in ymlItems:
Item().move(item, auxiliary)
Folder().save(folder)
# transform CSV if necessasry
csvItems = Folder().childItems(
folder,
filters={"lowerName": {
"$regex": csvRegex
}},
sort=[("created", pymongo.DESCENDING)],
)
if csvItems.count() >= 1:
file = Item().childFiles(csvItems.next())[0]
json_output = getTrackData(file)
saveTracks(folder, json_output, user)
csvItems.rewind()
for item in csvItems:
Item().move(item, auxiliary)
return folder
@access.user
@autoDescribeRoute(
Description("").jsonParam("data",
"",
requireObject=True,
paramType="body"))
def create_attribute(self, data, params):
attribute = Attribute().create(data["name"], data["belongs"],
data["datatype"], data["values"])
return attribute
@access.user
@autoDescribeRoute(Description(""))
def get_attributes(self):
return Attribute().find()
@access.user
@autoDescribeRoute(
Description("").modelParam("id", model=Attribute,
required=True).jsonParam("data",
"",
requireObject=True,
paramType="body"))
def update_attribute(self, data, attribute, params):
if "_id" in data:
del data["_id"]
attribute.update(data)
return Attribute().save(attribute)
@access.user
@autoDescribeRoute(
Description("").modelParam("id", model=Attribute, required=True))
def delete_attribute(self, attribute, params):
return Attribute().remove(attribute)
@access.user
@autoDescribeRoute(
Description("").modelParam(
"folderId",
description="folder id of a clip",
model=Folder,
paramType="query",
required=True,
level=AccessType.READ,
))
def get_valid_images(self, folder):
return Folder().childItems(
folder,
filters={"lowerName": {
"$regex": safeImageRegex
}},
sort=[("lowerName", pymongo.ASCENDING)],
)
class Container(Resource):
def initialize(self):
self.name = 'container'
self.exposeFields(level = AccessType.READ, fields = {'_id', 'status', 'error', 'ownerId', 'sessionId'})
def validate(self, container):
return container
@access.user
@filtermodel(model='container', plugin='wt_data_manager_testing')
@describeRoute(
Description('List containers for a given user.')
)
def listContainers(self, params):
user = self.getCurrentUser()
return list(self.model('container', 'wt_data_manager_testing').list(user=user))
@access.user
@filtermodel(model='container', plugin='wt_data_manager_testing')
@describeRoute(
Description('List containers for a given user.')
)
def listContainersFake(self, params):
return [{'_id': 1000, 'error': None}, {'_id': 1001, 'error': "Failed to start"}]
@access.user
@loadmodel(model='container', plugin='wt_data_manager_testing', level=AccessType.READ)
@describeRoute(
Description('Get a container by ID.')
.param('id', 'The ID of the container.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the container.', 403)
)
@filtermodel(model='container', plugin='wt_data_manager_testing')
def getContainer(self, container, params):
return container
@access.user
@loadmodel(model='container', plugin='wt_data_manager_testing', level=AccessType.WRITE)
@describeRoute(
Description('Stop an existing container.')
.param('id', 'The ID of the container.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Access was denied for the container.', 403)
)
def stopContainer(self, container, params):
user = self.getCurrentUser()
return self.model('container', 'wt_data_manager_testing').stopContainer(user, container)
@access.user
@loadmodel(model='container', plugin='wt_data_manager_testing', level=AccessType.WRITE)
@describeRoute(
Description('Starts a stopped container.')
.param('id', 'The ID of the container.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Access was denied for the container.', 403)
)
def startContainer(self, container, params):
user = self.getCurrentUser()
return self.model('container', 'wt_data_manager_testing').startContainer(user, container)
@access.user
@loadmodel(model='container', plugin='wt_data_manager_testing', level=AccessType.WRITE)
@describeRoute(
Description('Removes (and possibly stops) a container.')
.param('id', 'The ID of the container.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Access was denied for the container.', 403)
)
def removeContainer(self, container, params):
user = self.getCurrentUser()
return self.model('container', 'wt_data_manager_testing').removeContainer(user, container)
@access.user
@describeRoute(
Description('Starts a container.').
param('dataSet', 'An optional data set to initialize the container with. '
'A data set is a list of objects of the form '
'{"itemId": string, "mountPath": string, "externalUrl": string}.')
)
def createContainer(self, params):
user = self.getCurrentUser()
sDataSet = params.get('dataSet', '[]')
print("Data set param: " + str(sDataSet))
dataSet = json.loads(sDataSet)
return self.model('container', 'wt_data_manager_testing').createContainer(user, dataSet)
def stripQuotes(self, str):
if (str[0] == '\'' and str[-1] == '\'') or (str[0] == '"' and str[-1] == '"'):
return str[1:-1]
class ResourceExt(Resource):
def __init__(self, info):
"""
Initialize the resource. This saves the info path so that we can
dynamically change the routes when the setting listing the resources we
use changes.
:param info: the info class passed to the load function.
"""
super(ResourceExt, self).__init__()
self.loadInfo = info
self.boundResources = {}
# These methods implement the endpoint routing
def bindModels(self, event=None):
"""
When the list of tracked provenance resources is changed, rebind the
appropriate models to this instance of this class.
:param event: the event when a setting is saved, or None to check the
binding.
"""
if not event or event.name == 'provenance.initialize':
pass
elif event.name == 'model.setting.save.after':
if not hasattr(event, "info"):
return
if event.info.get(
'key',
'') != constants.PluginSettings.PROVENANCE_RESOURCES:
return
else:
return
resources = Setting().get(
constants.PluginSettings.PROVENANCE_RESOURCES)
if resources:
resources = resources.replace(',', ' ').strip().split()
else:
resources = []
resources = dict.fromkeys(resources)
# Always include item
resources['item'] = None
# Exclude resources that should never have provenance
for disallowedResource in ('model_base', 'notification', 'password',
'token'):
if disallowedResource in resources:
del resources[disallowedResource]
self.unbindModels(resources)
for resource in resources:
if resource not in self.boundResources:
events.bind('model.%s.save' % resource, 'provenance',
self.resourceSaveHandler)
events.bind('model.%s.copy.prepare' % resource, 'provenance',
self.resourceCopyHandler)
if hasattr(self.loadInfo['apiRoot'], resource):
getattr(self.loadInfo['apiRoot'],
resource).route('GET', (':id', 'provenance'),
self.getGetHandler(resource))
self.boundResources[resource] = True
def unbindModels(self, resources={}):
"""
Unbind any models that were bound and aren't listed as needed.
:param resources: resources that shouldn't be unbound.
"""
# iterate over a list so that we can change the dictionary as we use it
for oldresource in list(six.viewkeys(self.boundResources)):
if oldresource not in resources:
# Unbind this and remove it from the api
events.unbind('model.%s.save' % oldresource, 'provenance')
events.unbind('model.%s.copy.prepare' % oldresource,
'provenance')
if hasattr(self.loadInfo['apiRoot'], oldresource):
getattr(self.loadInfo['apiRoot'],
oldresource).removeRoute('GET',
(':id', 'provenance'))
del self.boundResources[oldresource]
def getGetHandler(self, resource):
"""
Return a function that will get the provenance for a particular
resource type. This creates such a function if necessary, copying the
main function and setting an internal value so that the function is
coupled to the resource.
:param resource: the name of the resource to get.
:returns: getHandler function.
"""
key = 'provenanceGetHandler_%s' % resource
if not hasattr(self, key):
def resourceGetHandler(id, params):
return self.provenanceGetHandler(id, params, resource)
# We inherit the description and access decorator details from the
# general provenanceGetHandler
for attr in ('description', 'accessLevel'):
setattr(resourceGetHandler, attr,
getattr(self.provenanceGetHandler, attr))
setattr(self, key, resourceGetHandler)
return getattr(self, key)
@access.public
@describeRoute(
Description('Get the provenance for a given resource.').param(
'id', 'The resource ID', paramType='path').param(
'version', 'The provenance version for the resource. If not '
'specified, the latest provenance data is returned. If "all" '
'is specified, a list of all provenance data is returned. '
'Negative indices can also be used (-1 is the latest '
'provenance, -2 second latest, etc.).',
required=False).errorResponse())
def provenanceGetHandler(self, id, params, resource=None):
user = self.getCurrentUser()
model = self.model(resource)
if isinstance(model,
(acl_mixin.AccessControlMixin, AccessControlledModel)):
obj = model.load(id, level=AccessType.READ, user=user)
else:
obj = model.load(id)
version = -1
if 'version' in params:
if params['version'] == 'all':
version = None
else:
try:
version = int(params['version'])
except ValueError:
raise RestException('Invalid version.')
provenance = obj.get('provenance', [])
result = None
if version is None or version == 0:
result = provenance
elif version < 0:
if len(provenance) >= -version:
result = provenance[version]
else:
for prov in provenance:
if prov.get('version', None) == version:
result = prov
break
return {'resourceId': id, 'provenance': result}
# These methods maintain the provenance
def resourceSaveHandler(self, event):
# get the resource name from the event
resource = event.name.split('.')[1]
obj = event.info
if ('_id' not in obj or
('provenance' not in obj
and obj.get('updated', None) == obj.get('created', 'unknown'))):
self.createNewProvenance(obj, resource)
else:
if 'provenance' not in obj:
self.createExistingProvenance(obj, resource)
elif (obj.get('updated', None) != obj['provenance'][-1].get(
'eventTime', False)):
self.updateProvenance(obj, resource)
def createNewProvenance(self, obj, resource):
provenance = []
created = obj.get('created', datetime.datetime.utcnow())
creatorId = obj.get('creatorId', None)
if creatorId is None:
user = self.getProvenanceUser(obj)
if user is not None:
creatorId = user['_id']
creationEvent = {
'eventType': 'creation',
'eventUser': creatorId,
'eventTime': obj.get('updated', created),
'created': created
}
obj['provenance'] = provenance
self.addProvenanceEvent(obj, creationEvent, resource)
def getProvenanceUser(self, obj):
"""
Get the user that is associated with the current provenance change.
This is the current session user, if there is one. If not, it is the
object's user or creator.
:param obj: a model object.
:returns: user for the object or None.
"""
user = self.getCurrentUser()
if obj and not user:
user = obj.get('userId', None)
if not user:
user = obj.get('creatorId', None)
if isinstance(user, tuple([ObjectId] + list(six.string_types))):
user = User().load(user, force=True)
return user
def createExistingProvenance(self, obj, resource):
self.createNewProvenance(obj, resource)
# we don't know what happened between creation and now
provenance = obj['provenance']
provenance[0]['eventType'] = 'unknownHistory'
# but we can track starting now
self.updateProvenance(obj, resource)
def addProvenanceEvent(self, obj, provenanceEvent, resource):
if 'provenance' not in obj:
self.createExistingProvenance(obj, resource)
provenance = obj['provenance']
self.incrementVersion(provenance, provenanceEvent)
provenance.append(provenanceEvent)
def incrementVersion(self, provenance, provenanceEvent):
if len(provenance) == 0:
# counting from 1, since people do that
provenanceEvent['version'] = 1
else:
provenanceEvent['version'] = int(provenance[-1]['version']) + 1
def updateProvenance(self, curObj, resource):
"""
Update the provenance record of an object.
:param curObj: the object to potentially update.
:param resource: the type of resource (model name).
:returns: True if the provenance was updated, False if it stayed the
same.
"""
user = self.getProvenanceUser(curObj)
model = self.model(resource)
if isinstance(model,
(acl_mixin.AccessControlMixin, AccessControlledModel)):
prevObj = model.load(curObj['_id'], force=True)
else:
prevObj = model.load(curObj['_id'])
if prevObj is None:
return False
oldData, newData = self.resourceDifference(prevObj, curObj)
if not len(newData) and not len(oldData):
return False
updateEvent = {
'eventType': 'update',
'eventTime': curObj.get('updated', datetime.datetime.utcnow()),
'new': newData,
'old': oldData
}
if user is not None:
updateEvent['eventUser'] = user['_id']
self.addProvenanceEvent(curObj, updateEvent, resource)
return True
def resourceDifference(self, prevObj, curObj):
"""
Generate dictionaries with values that have changed between two
objects.
:param prevObj: the initial state of the object.
:param curObj: the current state of the object.
:returns: oldData: a dictionary of values that are no longer the same
or present in the object.
:returns: newData: a dictionary of values that are different or new in
the object.
"""
curSnapshot = self.snapshotResource(curObj)
prevSnapshot = self.snapshotResource(prevObj)
oldData = {}
newData = {}
for key in curSnapshot:
if key in prevSnapshot:
try:
if curSnapshot[key] != prevSnapshot[key]:
newData[key] = curSnapshot[key]
oldData[key] = prevSnapshot[key]
except TypeError:
# If the data types of the old and new keys are not
# comparable, an error is thrown. In this case, always
# treat them as different.
newData[key] = curSnapshot[key]
oldData[key] = prevSnapshot[key]
else:
newData[key] = curSnapshot[key]
for key in prevSnapshot:
if key not in curSnapshot:
oldData[key] = prevSnapshot[key]
return oldData, newData
def snapshotResource(self, obj):
"""
Generate a dictionary that represents an arbitrary resource. All
fields are included except provenance and values starting with _.
:param obj: the object for which to generate a snapshop dictionary.
:param includeItemFiles: if True and this is an item, include files.
:returns: a snapshot dictionary.
"""
ignoredKeys = ('provenance', 'updated')
snap = {
key: obj[key]
for key in obj
if not key.startswith('_') and key not in ignoredKeys
}
return snap
def fileSaveHandler(self, event):
"""
When a file is saved, update the provenance of the parent item.
:param event: the event with the file information.
"""
curFile = event.info
if not curFile.get('itemId') or '_id' not in curFile:
return
user = self.getProvenanceUser(curFile)
item = Item().load(id=curFile['itemId'], force=True)
if not item:
return
prevFile = File().load(curFile['_id'], force=True)
if prevFile is None:
oldData = None
newData = self.snapshotResource(curFile)
else:
oldData, newData = self.resourceDifference(prevFile, curFile)
if not len(newData) and not len(oldData):
return
updateEvent = {
'eventType': 'fileUpdate',
'eventTime': curFile.get('updated', datetime.datetime.utcnow()),
'file': [{
'fileId': curFile['_id'],
'new': newData
}]
}
if oldData is not None:
updateEvent['file'][0]['old'] = oldData
if user is not None:
updateEvent['eventUser'] = user['_id']
self.addProvenanceEvent(item, updateEvent, 'item')
Item().save(item, triggerEvents=False)
def fileSaveCreatedHandler(self, event):
"""
When a file is created, we don't record it in the save handler
because we want to know its id. We record it here, instead.
:param event: the event with the file information.
"""
file = event.info
if not file.get('itemId') or '_id' not in file:
return
user = self.getProvenanceUser(file)
item = Item().load(id=file['itemId'], force=True)
if not item:
return
updateEvent = {
'eventType': 'fileAdded',
'eventTime': file.get('created', datetime.datetime.utcnow()),
'file': [{
'fileId': file['_id'],
'new': self.snapshotResource(file)
}]
}
if user is not None:
updateEvent['eventUser'] = user['_id']
self.addProvenanceEvent(item, updateEvent, 'item')
Item().save(item, triggerEvents=False)
def fileRemoveHandler(self, event):
"""
When a file is removed, update the provenance of the parent item.
:param event: the event with the file information.
"""
file = event.info
itemId = file.get('itemId')
# Don't attach provenance to an item based on files that are not
# directly associated (we may want to revisit this and, when files are
# attachedToType, add provenance to the appropriate type and ID).
if not itemId:
return
user = self.getProvenanceUser(file)
item = Item().load(id=itemId, force=True)
if not item:
return
updateEvent = {
'eventType': 'fileRemoved',
'eventTime': datetime.datetime.utcnow(),
'file': [{
'fileId': file['_id'],
'old': self.snapshotResource(file)
}]
}
if user is not None:
updateEvent['eventUser'] = user['_id']
self.addProvenanceEvent(item, updateEvent, 'item')
Item().save(item, triggerEvents=False)
def resourceCopyHandler(self, event):
# Use the old item's provenance, but add a copy record.
resource = event.name.split('.')[1]
srcObj, newObj = event.info
# We should have marked the new object already when it was first
# created. If not, exit
if 'provenance' not in newObj:
pass # pragma: no cover
if 'provenance' in srcObj:
newProv = newObj['provenance'][-1]
newObj['provenance'] = copy.deepcopy(srcObj['provenance'])
newProv['version'] = newObj['provenance'][-1]['version'] + 1
newObj['provenance'].append(newProv)
# Convert the creation record to a copied record
newObj['provenance'][-1]['eventType'] = 'copy'
if '_id' in srcObj:
newObj['provenance'][-1]['originalId'] = srcObj['_id']
self.model(resource).save(newObj, triggerEvents=False)
class ImageBrowseResource(ItemResource):
"""Extends the "item" resource to iterate through images im a folder."""
def __init__(self, apiRoot):
# Don't call the parent (Item) constructor, to avoid redefining routes,
# but do call the grandparent (Resource) constructor
super(ItemResource, self).__init__()
self.resourceName = 'item'
apiRoot.item.route('GET', (':id', 'next_image'), self.getNextImage)
apiRoot.item.route('GET', (':id', 'previous_image'), self.getPreviousImage)
def getAdjacentImages(self, currentImage, currentFolder=None):
folderModel = Folder()
if currentFolder:
folder = currentFolder
else:
folder = folderModel.load(
currentImage['folderId'], user=self.getCurrentUser(), level=AccessType.READ)
if folder.get('isVirtual'):
children = folderModel.childItems(folder, includeVirtual=True)
else:
children = folderModel.childItems(folder)
allImages = [item for item in children if _isLargeImageItem(item)]
try:
index = allImages.index(currentImage)
except ValueError:
raise RestException('Id is not an image', 404)
return {
'previous': allImages[index - 1],
'next': allImages[(index + 1) % len(allImages)]
}
@access.public
@autoDescribeRoute(
Description('Get the next image in the same folder as the given item.')
.modelParam('id', 'The current image ID',
model='item', destName='image', paramType='path', level=AccessType.READ)
.modelParam('folderId', 'The (virtual) folder ID the image is located in',
model='folder', destName='folder', paramType='query', level=AccessType.READ,
required=False)
.errorResponse()
.errorResponse('Image not found', code=404)
)
def getNextImage(self, image, folder):
return self.getAdjacentImages(image, folder)['next']
@access.public
@autoDescribeRoute(
Description('Get the previous image in the same folder as the given item.')
.modelParam('id', 'The current item ID',
model='item', destName='image', paramType='path', level=AccessType.READ)
.modelParam('folderId', 'The (virtual) folder ID the image is located in',
model='folder', destName='folder', paramType='query', level=AccessType.READ,
required=False)
.errorResponse()
.errorResponse('Image not found', code=404)
)
def getPreviousImage(self, image, folder):
return self.getAdjacentImages(image, folder)['previous']
class CeleryTestEndpoints(Resource):
def __init__(self):
super(CeleryTestEndpoints, self).__init__()
self.route('POST', ('test_task_delay', ), self.test_celery_task_delay)
self.route('POST', ('test_task_delay_fails', ),
self.test_celery_task_delay_fails)
self.route('POST', ('test_task_apply_async', ),
self.test_celery_task_apply_async)
self.route('POST', ('test_task_apply_async_fails', ),
self.test_celery_task_apply_async_fails)
self.route('POST', ('test_task_signature_delay', ),
self.test_celery_task_signature_delay)
self.route('POST', ('test_task_signature_delay_fails', ),
self.test_celery_task_signature_delay_fails)
self.route('POST', ('test_task_signature_apply_async', ),
self.test_celery_task_signature_apply_async)
self.route('POST', ('test_task_signature_apply_async_fails', ),
self.test_celery_task_signature_apply_async_fails)
self.route('POST', ('test_task_revoke', ),
self.test_celery_task_revoke)
self.route('POST', ('test_task_revoke_in_queue', ),
self.test_celery_task_revoke_in_queue)
self.route('POST', ('test_task_delay_with_custom_job_options', ),
self.test_celery_task_delay_with_custom_job_options)
self.route('POST', ('test_task_apply_async_with_custom_job_options', ),
self.test_celery_task_apply_async_with_custom_job_options)
self.route('POST', ('test_girder_client_generation', ),
self.test_celery_girder_client_generation)
self.route('POST', ('test_girder_client_bad_token_fails', ),
self.test_celery_girder_client_bad_token_fails)
# Testing custom job option API for celery tasks
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test celery task delay with custom job options'))
def test_celery_task_delay_with_custom_job_options(self, params):
result = fibonacci.delay(20, girder_job_title='TEST DELAY TITLE')
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test celery task apply_async with custom job options'))
def test_celery_task_apply_async_with_custom_job_options(self, params):
result = fibonacci.apply_async(
(20, ), {}, girder_job_title='TEST APPLY_ASYNC TITLE')
return result.job
# Testing basic celery API
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task delay'))
def test_celery_task_delay(self, params):
result = fibonacci.delay(20)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task delay fails correctly'))
def test_celery_task_delay_fails(self, params):
result = fail_after.delay()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task apply_async'))
def test_celery_task_apply_async(self, params):
result = fibonacci.apply_async((20, ))
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task apply_async fails correctly'))
def test_celery_task_apply_async_fails(self, params):
result = fail_after.apply_async((0.5, ))
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task signature delay'))
def test_celery_task_signature_delay(self, params):
signature = fibonacci.s(20)
result = signature.delay()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test celery task signature delay fails correctly'))
def test_celery_task_signature_delay_fails(self, params):
signature = fail_after.s(0.5)
result = signature.delay()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task apply_async'))
def test_celery_task_signature_apply_async(self, params):
signature = fibonacci.s(20)
result = signature.apply_async()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test celery task apply_async fails correctly'))
def test_celery_task_signature_apply_async_fails(self, params):
signature = fail_after.s(0.5)
result = signature.apply_async()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description(
'Test girder client is generated and can request scoped endpoints')
)
def test_celery_girder_client_generation(self, params):
token = ModelImporter.model('token').createToken(
user=self.getCurrentUser())
result = request_private_path.delay('admin',
girder_client_token=str(
token['_id']))
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description(
'Test girder client with no token can\'t access protected resources'
))
def test_celery_girder_client_bad_token_fails(self, params):
result = request_private_path.delay('admin', girder_client_token='')
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test revoking a task directly'))
def test_celery_task_revoke(self, params):
result = cancelable.delay()
# Make sure we are running before we revoke
assert wait_for_status(self.getCurrentUser(), result.job,
JobStatus.RUNNING)
result.revoke()
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(Description('Test revoking a task directly when in queue'))
def test_celery_task_revoke_in_queue(self, params):
# Fill up queue
blockers = []
for _ in range(0, multiprocessing.cpu_count()):
blockers.append(cancelable.delay(sleep_interval=0.1))
result = cancelable.delay()
result.revoke()
assert wait_for_status(self.getCurrentUser(), result.job,
JobStatus.CANCELED)
# Now clean up the blockers
for blocker in blockers:
blocker.revoke()
return result.job
file['imported'] = True
file['pathInTarfile'] = entry.name
File().save(file)
@boundHandler
@access.admin(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Archive the contents of a Girder folder.')
.notes('This will move and (optionally compress) files from their assetstore location '
'to within a tape archive (tar) file. They can still be served from that file as '
'usual, but there will be a latency associated with reading from the archive format. '
'\n\nFiles under this folder that are already archived will be skipped and not '
'added to the new archive.')
.modelParam('id', model=Assetstore)
.modelParam('folderId', 'The folder to archive.', model=Folder, level=AccessType.WRITE,
paramType='formData')
.param('path', 'Path where the tar file will be written, relative to assetstore root.')
.param('compression', 'Compression method', required=False, default='gz',
enum=('gz', 'bz2', ''))
.param('progress', 'Whether to record progress on the import.',
dataType='boolean', default=False, required=False)
.errorResponse()
.errorResponse('You are not an administrator.', 403))
def _exportTar(self, assetstore, folder, path, compression, progress):
user = self.getCurrentUser()
adapter = getAssetstoreAdapter(assetstore)
with ProgressContext(progress, user=user, title='Archiving %s' % folder['name']) as ctx:
adapter._exportTar(path, folder, ctx, user, compression)
class WebClientTestEndpoints(Resource):
def __init__(self):
super(WebClientTestEndpoints, self).__init__()
self.route('GET', ('progress', ), self.testProgress)
self.route('PUT', ('progress', 'stop'), self.testProgressStop)
self.route('POST', ('file', ), self.uploadFile)
self.stop = False
@access.token
@describeRoute(
Description('Test progress contexts from the web').param(
'test', 'Name of test to run. These include "success" and '
'"failure".',
required=False).param('duration',
'Duration of the test in seconds',
required=False,
dataType='int'))
def testProgress(self, params):
test = params.get('test', 'success')
duration = int(params.get('duration', 10))
startTime = time.time()
with ProgressContext(True,
user=self.getCurrentUser(),
title='Progress Test',
message='Progress Message',
total=duration) as ctx:
for current in range(duration):
if self.stop:
break
ctx.update(current=current)
wait = startTime + current + 1 - time.time()
if wait > 0:
time.sleep(wait)
if test == 'error':
raise RestException('Progress error test.')
@access.token
@describeRoute(Description('Halt all progress tests'))
def testProgressStop(self, params):
self.stop = True
@access.user
@describeRoute(None)
def uploadFile(self, params):
"""
Providing this works around a limitation in phantom that makes us
unable to upload binary files, or at least ones that contain certain
byte values. The path parameter should be provided relative to the
root directory of the repository.
"""
self.requireParams(('folderId', 'path'), params)
path = os.path.join(ROOT_DIR, params['path'])
name = os.path.basename(path)
folder = self.model('folder').load(params['folderId'], force=True)
upload = self.model('upload').createUpload(user=self.getCurrentUser(),
name=name,
parentType='folder',
parent=folder,
size=os.path.getsize(path))
with open(path, 'rb') as fd:
file = self.model('upload').handleChunk(upload, fd)
return file
reuseExisting=True)
gc_item = itemModel.setMetadata(gc_item,
{'identifier': fileObj['identifier']})
fileModel.createLinkFile(url=fileObj['url'],
parent=gc_item,
name=fileName,
parentType='item',
creator=user,
size=int(fileObj['size']),
mimeType=fileObj['formatId'],
reuseExisting=True)
# Recurse and add child packages if any exist
if children is not None and len(children) > 0:
for child in children:
register_DataONE_resource(gc_folder, 'folder', progress, user,
child['identifier'])
return gc_folder
@access.user(scope=TokenScope.DATA_READ)
@filtermodel(model='folder')
@autoDescribeRoute(
Description('List all folders containing references to an external data').
errorResponse('Write access denied for parent collection.', 403))
@boundHandler()
def listImportedData(self, params):
q = {'meta.provider': {'$exists': 1}}
return list(ModelImporter.model('folder').find(query=q))
class HistomicsTKResource(DockerResource):
def __init__(self, name, *args, **kwargs):
super(HistomicsTKResource, self).__init__(name, *args, **kwargs)
self.route('GET', ('settings', ), self.getPublicSettings)
self.route('PUT', ('quarantine', ':id'), self.putQuarantine)
self.route('PUT', ('quarantine', ':id', 'restore'),
self.restoreQuarantine)
self.route('GET', ('analysis', 'access'), self.getAnalysisAccess)
def _accessList(self):
access = Setting().get(
PluginSettings.HISTOMICSTK_ANALYSIS_ACCESS) or {}
acList = {
'users': [{
'id': x,
'level': AccessType.READ
} for x in access.get('users', [])],
'groups': [{
'id': x,
'level': AccessType.READ
} for x in access.get('groups', [])],
'public':
access.get('public', True),
}
for user in acList['users'][:]:
userDoc = User().load(user['id'],
force=True,
fields=['firstName', 'lastName', 'login'])
if userDoc is None:
acList['users'].remove(user)
else:
user['login'] = userDoc['login']
user['name'] = ' '.join(
(userDoc['firstName'], userDoc['lastName']))
for grp in acList['groups'][:]:
grpDoc = Group().load(grp['id'],
force=True,
fields=['name', 'description'])
if grpDoc is None:
acList['groups'].remove(grp)
else:
grp['name'] = grpDoc['name']
grp['description'] = grpDoc['description']
return acList
@access.public
@describeRoute(Description('List docker images and their CLIs'))
def getDockerImages(self, *args, **kwargs):
user = self.getCurrentUser()
if not user:
return {}
result = super(HistomicsTKResource,
self).getDockerImages(*args, **kwargs)
acList = self._accessList()
# Use the User().hasAccess to test for access via a synthetic document
if not User().hasAccess({
'access': acList,
'public': acList['public']
}, user):
return {}
return result
@describeRoute(Description('Get public settings for HistomicsTK.'))
@access.public
def getPublicSettings(self, params):
keys = [
PluginSettings.HISTOMICSTK_DEFAULT_DRAW_STYLES,
PluginSettings.HISTOMICSTK_QUARANTINE_FOLDER,
]
result = {k: self.model('setting').get(k) for k in keys}
result[PluginSettings.HISTOMICSTK_QUARANTINE_FOLDER] = bool(
result[PluginSettings.HISTOMICSTK_QUARANTINE_FOLDER])
return result
@describeRoute(Description('Get the access list for analyses.'))
@access.admin
def getAnalysisAccess(self, params):
return self._accessList()
@autoDescribeRoute(
Description('Move an item to the quarantine folder.').responseClass(
'Item').modelParam('id', model=Item,
level=AccessType.WRITE).errorResponse(
'ID was invalid.').errorResponse(
'Write access was denied for the item',
403))
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=Item)
def putQuarantine(self, item):
folder = Setting().get(PluginSettings.HISTOMICSTK_QUARANTINE_FOLDER)
if not folder:
raise RestException('The quarantine folder is not configured.')
folder = Folder().load(folder, force=True, exc=True)
if not folder:
raise RestException('The quarantine folder does not exist.')
if str(folder['_id']) == str(item['folderId']):
raise RestException(
'The item is already in the quarantine folder.')
originalFolder = Folder().load(item['folderId'], force=True)
quarantineInfo = {
'originalFolderId': item['folderId'],
'originalBaseParentType': item['baseParentType'],
'originalBaseParentId': item['baseParentId'],
'originalUpdated': item['updated'],
'quarantineUserId': self.getCurrentUser()['_id'],
'quarantineTime': datetime.datetime.utcnow()
}
item = Item().move(item, folder)
placeholder = Item().createItem(item['name'],
{'_id': item['creatorId']},
originalFolder,
description=item['description'])
quarantineInfo['placeholderItemId'] = placeholder['_id']
item.setdefault('meta', {})['quarantine'] = quarantineInfo
item = Item().updateItem(item)
placeholderInfo = {
'quarantined': True,
'quarantineTime': quarantineInfo['quarantineTime']
}
placeholder.setdefault('meta', {})['quarantine'] = placeholderInfo
placeholder = Item().updateItem(placeholder)
return item
@autoDescribeRoute(
Description('Restore a quarantined item to its original folder.').
responseClass('Item').modelParam(
'id', model=Item, level=AccessType.WRITE).errorResponse(
'ID was invalid.').errorResponse(
'Write access was denied for the item', 403))
@access.admin
@filtermodel(model=Item)
def restoreQuarantine(self, item):
if not item.get('meta', {}).get('quarantine'):
raise RestException('The item has no quarantine record.')
folder = Folder().load(item['meta']['quarantine']['originalFolderId'],
force=True)
if not folder:
raise RestException('The original folder is not accesible.')
placeholder = Item().load(
item['meta']['quarantine']['placeholderItemId'], force=True)
item = Item().move(item, folder)
item['updated'] = item['meta']['quarantine']['originalUpdated']
del item['meta']['quarantine']
item = Item().updateItem(item)
if placeholder is not None:
Item().remove(placeholder)
return item
class Cluster(BaseResource):
def __init__(self):
super(Cluster, self).__init__()
self.resourceName = 'clusters'
self.route('POST', (), self.create)
self.route('POST', (':id', 'log'), self.handle_log_record)
self.route('GET', (':id', 'log'), self.log)
self.route('PUT', (':id', 'start'), self.start)
self.route('PUT', (':id', 'launch'), self.launch)
self.route('PUT', (':id', 'provision'), self.provision)
self.route('PATCH', (':id', ), self.update)
self.route('GET', (':id', 'status'), self.status)
self.route('PUT', (':id', 'terminate'), self.terminate)
self.route('PUT', (':id', 'job', ':jobId', 'submit'), self.submit_job)
self.route('GET', (':id', ), self.get)
self.route('DELETE', (':id', ), self.delete)
self.route('GET', (), self.find)
# TODO Findout how to get plugin name rather than hardcoding it
self._model = ModelImporter.model('cluster', 'cumulus')
@access.user(scope=TokenScope.DATA_WRITE)
def handle_log_record(self, id, params):
user = self.getCurrentUser()
if not self._model.load(id, user=user, level=AccessType.ADMIN):
raise RestException('Cluster not found.', code=404)
return self._model.append_to_log(user, id, getBodyJson())
handle_log_record.description = None
def _create_ec2(self, params, body):
return self._create_ansible(params, body, cluster_type=ClusterType.EC2)
def _create_ansible(self, params, body, cluster_type=ClusterType.ANSIBLE):
self.requireParams(['name', 'profileId'], body)
name = body['name']
playbook = get_property('config.launch.spec', body, default='default')
launch_params = get_property('config.launch.params', body, default={})
config = get_property('config', body, default={})
profile_id = body['profileId']
user = self.getCurrentUser()
cluster = self._model.create_ansible(user,
name,
config,
playbook,
launch_params,
profile_id,
cluster_type=cluster_type)
return cluster
def _create_traditional(self, params, body):
self.requireParams(['name', 'config'], body)
self.requireParams(['ssh', 'host'], body['config'])
self.requireParams(['user'], body['config']['ssh'])
name = body['name']
config = body['config']
user = self.getCurrentUser()
cluster = self._model.create_traditional(user, name, config)
# Fire off job to create key pair for cluster
girder_token = self.get_task_token()['_id']
generate_key_pair.delay(self._model.filter(cluster, user),
girder_token)
return cluster
def _create_newt(self, params, body):
self.requireParams(['name', 'config'], body)
self.requireParams(['host'], body['config'])
name = body['name']
config = body['config']
user = self.getCurrentUser()
config['user'] = user['login']
cluster = self._model.create_newt(user, name, config)
return cluster
@access.user(scope=TokenScope.DATA_WRITE)
def create(self, params):
body = getBodyJson()
# Default ec2 cluster
cluster_type = 'ec2'
if 'type' in body:
if not ClusterType.is_valid_type(body['type']):
raise RestException('Invalid cluster type.', code=400)
cluster_type = body['type']
if cluster_type == ClusterType.EC2:
cluster = self._create_ec2(params, body)
elif cluster_type == ClusterType.ANSIBLE:
cluster = self._create_ansible(params, body)
elif cluster_type == ClusterType.TRADITIONAL:
cluster = self._create_traditional(params, body)
elif cluster_type == ClusterType.NEWT:
cluster = self._create_newt(params, body)
else:
raise RestException('Invalid cluster type.', code=400)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] = '/clusters/%s' % cluster['_id']
return self._model.filter(cluster, self.getCurrentUser())
addModel(
'Id', {
'id': 'Id',
'properties': {
'_id': {
'type': 'string',
'description': 'The id.'
}
}
}, 'clusters')
addModel(
'UserNameParameter', {
'id': 'UserNameParameter',
'properties': {
'user': {
'type': 'string',
'description': 'The ssh user id'
}
}
}, 'clusters')
addModel(
'SshParameters', {
'id': 'SshParameters',
'properties': {
'ssh': {
'$ref': '#/definitions/UserNameParameter'
}
}
}, 'clusters')
addModel(
'ClusterParameters', {
'id': 'ClusterParameters',
'required': ['name', 'config', 'type'],
'properties': {
'name': {
'type': 'string',
'description': 'The name to give the cluster.'
},
'template': {
'type': 'string',
'description': 'The cluster template to use. '
'(ec2 only)'
},
'config': {
'$ref': '#/definitions/SshParameters',
'host': {
'type':
'string',
'description':
'The hostname of the head node '
'(trad only)'
}
},
'type': {
'type': 'string',
'description': 'The cluster type, either "ec2" or "trad"'
}
}
}, 'clusters')
create.description = (Description('Create a cluster').param(
'body',
'The name to give the cluster.',
dataType='ClusterParameters',
required=True,
paramType='body'))
def _get_body(self):
body = {}
if cherrypy.request.body:
request_body = cherrypy.request.body.read().decode('utf8')
if request_body:
body = json.loads(request_body)
return body
@access.user(scope=TokenScope.DATA_WRITE)
@loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
def start(self, cluster, params):
body = self._get_body()
adapter = get_cluster_adapter(cluster)
adapter.start(body)
events.trigger('cumulus.cluster.started', info=cluster)
addModel(
'ClusterOnStartParms', {
'id': 'ClusterOnStartParms',
'properties': {
'submitJob': {
'pattern':
'^[0-9a-fA-F]{24}$',
'type':
'string',
'description':
'The id of a Job to submit when the cluster '
'is started.'
}
}
}, 'clusters')
addModel(
'ClusterStartParams', {
'id': 'ClusterStartParams',
'properties': {
'onStart': {
'$ref': '#/definitions/ClusterOnStartParms'
}
}
}, 'clusters')
start.description = (Description('Start a cluster (ec2 only)').param(
'id', 'The cluster id to start.', paramType='path',
required=True).param('body',
'Parameter used when starting cluster',
paramType='body',
dataType='ClusterStartParams',
required=False))
@access.user(scope=TokenScope.DATA_WRITE)
@loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
def launch(self, cluster, params):
# Update any launch parameters passed in message body
body = self._get_body()
cluster['config']['launch']['params'].update(body)
cluster = self._model.save(cluster)
return self._model.filter(self._launch_or_provision('launch', cluster),
self.getCurrentUser())
launch.description = (Description('Start a cluster with ansible').param(
'id', 'The cluster id to start.', paramType='path', required=True))
@access.user(scope=TokenScope.DATA_WRITE)
@loadmodel(model='cluster', plugin='cumulus', level=AccessType.ADMIN)
def provision(self, cluster, params):
if not ClusterStatus.valid_transition(cluster['status'],
ClusterStatus.PROVISIONING):
raise RestException(
'Cluster status is %s and cannot be provisioned' %
cluster['status'],
code=400)
body = self._get_body()
provision_ssh_user = get_property('ssh.user', body)
if provision_ssh_user:
cluster['config'].setdefault('provision', {})['ssh'] = {
'user': provision_ssh_user
}
del body['ssh']
if 'spec' in body:
cluster['config'].setdefault('provision', {})['spec'] \
= body['spec']
del body['spec']
cluster['config'].setdefault('provision', {})\
.setdefault('params', {}).update(body)
cluster = self._model.save(cluster)
return self._model.filter(
self._launch_or_provision('provision', cluster),
self.getCurrentUser())
provision.description = (
Description('Provision a cluster with ansible').param(
'id',
'The cluster id to provision.',
paramType='path',
required=True).param('body',
'Parameter used when provisioning cluster',
paramType='body',
dataType='list',
required=False))
def _launch_or_provision(self, process, cluster):
assert process in ['launch', 'provision']
adapter = get_cluster_adapter(cluster)
return getattr(adapter, process)()
@access.user(scope=TokenScope.DATA_WRITE)
def update(self, id, params):
body = getBodyJson()
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.WRITE)
if not cluster:
raise RestException('Cluster not found.', code=404)
if 'assetstoreId' in body:
cluster['assetstoreId'] = body['assetstoreId']
if 'status' in body:
if ClusterStatus.valid(body['status']):
cluster['status'] = body['status']
else:
raise RestException('%s is not a valid cluster status' %
body['status'],
code=400)
if 'timings' in body:
if 'timings' in cluster:
cluster['timings'].update(body['timings'])
else:
cluster['timings'] = body['timings']
if 'config' in body:
# Need to check we aren't try to update immutable fields
immutable_paths = ['_id', 'ssh.user']
for path in immutable_paths:
if parse(path).find(body['config']):
raise RestException("The '%s' field can't be updated" %
path)
update_dict(cluster['config'], body['config'])
cluster = self._model.update_cluster(user, cluster)
# Now do any updates the adapter provides
adapter = get_cluster_adapter(cluster)
try:
adapter.update(body)
# Skip adapter.update if update not defined for this adapter
except (NotImplementedError, ValidationException):
pass
return self._model.filter(cluster, user)
addModel(
'ClusterUpdateParameters', {
'id': 'ClusterUpdateParameters',
'properties': {
'status': {
'type': 'string',
'enum': ['created', 'running', 'stopped', 'terminated'],
'description': 'The new status. (optional)'
}
}
}, 'clusters')
update.description = (Description('Update the cluster').param(
'id', 'The id of the cluster to update', paramType='path').param(
'body',
'The properties to update.',
dataType='ClusterUpdateParameters',
paramType='body').notes('Internal - Used by Celery tasks'))
@access.user(scope=TokenScope.DATA_READ)
def status(self, id, params):
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.READ)
if not cluster:
raise RestException('Cluster not found.', code=404)
return {'status': cluster['status']}
addModel(
'ClusterStatus', {
'id': 'ClusterStatus',
'required': ['status'],
'properties': {
'status': {
'type': 'string',
'enum': [ClusterStatus.valid_transitions.keys()]
}
}
}, 'clusters')
status.description = (Description('Get the clusters current state').param(
'id', 'The cluster id to get the status of.',
paramType='path').responseClass('ClusterStatus'))
@access.user(scope=TokenScope.DATA_WRITE)
def terminate(self, id, params):
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.ADMIN)
if not cluster:
raise RestException('Cluster not found.', code=404)
adapter = get_cluster_adapter(cluster)
adapter.terminate()
terminate.description = (Description('Terminate a cluster').param(
'id', 'The cluster to terminate.', paramType='path'))
@access.user(scope=TokenScope.DATA_READ)
def log(self, id, params):
user = self.getCurrentUser()
offset = 0
if 'offset' in params:
offset = int(params['offset'])
if not self._model.load(id, user=user, level=AccessType.READ):
raise RestException('Cluster not found.', code=404)
log_records = self._model.log_records(user, id, offset)
return {'log': log_records}
log.description = (Description('Get log entries for cluster').param(
'id', 'The cluster to get log entries for.',
paramType='path').param('offset',
'The offset to start getting entries at.',
required=False,
paramType='query'))
@access.user(scope=TokenScope.DATA_WRITE)
def submit_job(self, id, jobId, params):
job_id = jobId
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.ADMIN)
if not cluster:
raise RestException('Cluster not found.', code=404)
if cluster['status'] != ClusterStatus.RUNNING:
raise RestException('Cluster is not running', code=400)
job_model = ModelImporter.model('job', 'cumulus')
job = job_model.load(job_id, user=user, level=AccessType.ADMIN)
# Set the clusterId on the job for termination
job['clusterId'] = ObjectId(id)
# Add any job parameters to be used when templating job script
body = cherrypy.request.body.read().decode('utf8')
if body:
job['params'] = json.loads(body)
job_model.save(job)
cluster_adapter = get_cluster_adapter(cluster)
del job['access']
del job['log']
cluster_adapter.submit_job(job)
submit_job.description = (Description('Submit a job to the cluster').param(
'id',
'The cluster to submit the job to.',
required=True,
paramType='path').param('jobId',
'The cluster to get log entries for.',
required=True,
paramType='path').param(
'body',
'The properties to template on submit.',
dataType='object',
paramType='body'))
@access.user(scope=TokenScope.DATA_READ)
def get(self, id, params):
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.ADMIN)
if not cluster:
raise RestException('Cluster not found.', code=404)
return self._model.filter(cluster, user)
get.description = (Description('Get a cluster').param('id',
'The cluster id.',
paramType='path',
required=True))
@access.user(scope=TokenScope.DATA_WRITE)
def delete(self, id, params):
user = self.getCurrentUser()
cluster = self._model.load(id, user=user, level=AccessType.ADMIN)
if not cluster:
raise RestException('Cluster not found.', code=404)
adapter = get_cluster_adapter(cluster)
adapter.delete()
self._model.delete(user, id)
delete.description = (
Description('Delete a cluster and its configuration').param(
'id', 'The cluster id.', paramType='path', required=True))
@access.user(scope=TokenScope.DATA_READ)
def find(self, params):
return self._model.find_cluster(params, user=self.getCurrentUser())
find.description = (
Description('Search for clusters with certain properties').param(
'type',
'The cluster type to search for',
paramType='query',
required=False).param('limit',
'The max number of clusters to return',
paramType='query',
required=False,
default=50))
class FeatureInfo(Resource):
def __init__(self):
self.resourceName = 'minerva_get_feature_info'
self.route('GET', (), self.getFeatureInfo)
def _getMinervaItem(self, itemId):
"""Returns minerva metadata for a given item_id"""
item = self.model('item').load(itemId, user=self.getCurrentUser())
return item
@staticmethod
def callFeatureInfo(baseUrl, params, typeNames):
"""Calls geoserver to get at long location information"""
baseUrl = baseUrl.replace('GetCapabilities', 'GetFeatureInfo')
typeNames = ",".join(typeNames)
parameters = {
'exceptions': 'application/vnd.ogc.se_xml',
'feature_count': '50',
'styles': '',
'srs': 'EPSG:3857',
'info_format': 'application/json',
'format': 'image/png',
'query_layers': typeNames,
'layers': typeNames,
'bbox': params['bbox'],
'width': params['width'],
'height': params['height'],
'x': params['x'],
'y': params['y'],
'callback': 'getLayerFeatures'
}
req = requests.get(baseUrl, params=parameters)
return req.content
@access.user
def getFeatureInfo(self, params):
activeLayers = params['activeLayers[]']
# Return a list for all cases
if isinstance(activeLayers, (str, unicode)):
activeLayers = [activeLayers]
layerSource = []
for i in activeLayers:
item = self._getMinervaItem(i)
url = item['meta']['minerva'].get('base_url')
layerSource.append((url, item['meta']['minerva']['type_name']))
layerUrlMap = defaultdict(list)
for k, v in layerSource:
layerUrlMap[k].append(v)
grandResponse = []
for baseUrl, layers in layerUrlMap.items():
event = events.trigger('minerva.get_layer_info', {
'baseUrl': baseUrl,
'params': params,
'layers': layers
})
response = event.responses
if not event.defaultPrevented:
response = self.callFeatureInfo(baseUrl, params, layers)
grandResponse.append(response)
return grandResponse
getFeatureInfo.description = (Description(
'Query values for overlayed datasets for a given lat long').param(
'activeLayers',
'Active layers on map').param('bbox', 'Bounding box').param(
'x', 'X',
dataType='int').param('y', 'Y', dataType='int').param(
'width', 'Width', dataType='int').param('height',
'Height',
dataType='int'))
class LargeImageResource(Resource):
def __init__(self):
super(LargeImageResource, self).__init__()
self.resourceName = 'large_image'
self.route('GET', ('thumbnails',), self.countThumbnails)
self.route('PUT', ('thumbnails',), self.createThumbnails)
self.route('DELETE', ('thumbnails',), self.deleteThumbnails)
@describeRoute(
Description('Count the number of cached thumbnail files for '
'large_image items.')
.param('spec', 'A JSON list of thumbnail specifications to count. '
'If empty, all cached thumbnails are counted. The '
'specifications typically include width, height, encoding, and '
'encoding options.', required=False)
)
@access.admin
def countThumbnails(self, params):
spec = params.get('spec')
if spec is not None:
try:
spec = json.loads(spec)
if not isinstance(spec, list):
raise ValueError()
except ValueError:
raise RestException('The spec parameter must be a JSON list.')
spec = [json.dumps(entry, sort_keys=True, separators=(',', ':'))
for entry in spec]
else:
spec = [None]
count = 0
for entry in spec:
query = {'isLargeImageThumbnail': True, 'attachedToType': 'item'}
if entry is not None:
query['thumbnailKey'] = entry
count += self.model('file').find(query).count()
return count
@describeRoute(
Description('Create cached thumbnail files from large_image items.')
.notes('This creates a local job that processes all large_image items.')
.param('spec', 'A JSON list of thumbnail specifications to create. '
'The specifications typically include width, height, encoding, '
'and encoding options.')
.param('logInterval', 'The number of seconds between log messages. '
'This also determines how often the creation job is checked if '
'it has been canceled or deleted. A value of 0 will log after '
'each thumbnail is checked or created.', required=False,
dataType='float')
)
@access.admin
def createThumbnails(self, params):
self.requireParams(['spec'], params)
try:
spec = json.loads(params['spec'])
if not isinstance(spec, list):
raise ValueError()
except ValueError:
raise RestException('The spec parameter must be a JSON list.')
maxThumbnailFiles = int(self.model('setting').get(
constants.PluginSettings.LARGE_IMAGE_MAX_THUMBNAIL_FILES))
if maxThumbnailFiles <= 0:
raise RestException('Thumbnail files are not enabled.')
jobModel = self.model('job', 'jobs')
jobKwargs = {'spec': spec}
if params.get('logInterval') is not None:
jobKwargs['logInterval'] = float(params['logInterval'])
job = jobModel.createLocalJob(
module='girder.plugins.large_image.rest.large_image',
function='createThumbnailsJob',
kwargs=jobKwargs,
title='Create large image thumbnail files.',
type='large_image_create_thumbnails',
user=self.getCurrentUser(),
public=True,
async=True,
)
jobModel.scheduleJob(job)
return job
@describeRoute(
Description('Delete cached thumbnail files from large_image items.')
.param('spec', 'A JSON list of thumbnail specifications to delete. '
'If empty, all cached thumbnails are deleted. The '
'specifications typically include width, height, encoding, and '
'encoding options.', required=False)
)
@access.admin
def deleteThumbnails(self, params):
spec = params.get('spec')
if spec is not None:
try:
spec = json.loads(spec)
if not isinstance(spec, list):
raise ValueError()
except ValueError:
raise RestException('The spec parameter must be a JSON list.')
spec = [json.dumps(entry, sort_keys=True, separators=(',', ':'))
for entry in spec]
else:
spec = [None]
removed = 0
for entry in spec:
query = {'isLargeImageThumbnail': True, 'attachedToType': 'item'}
if entry is not None:
query['thumbnailKey'] = entry
for file in self.model('file').find(query):
self.model('file').remove(file)
removed += 1
return removed
class DockerTestEndpoints(Resource):
def __init__(self):
super(DockerTestEndpoints, self).__init__()
self.route('POST', ('test_docker_run', ),
self.test_docker_run)
self.route('POST', ('test_docker_run_mount_volume', ),
self.test_docker_run_mount_volume)
self.route('POST', ('test_docker_run_named_pipe_output', ),
self.test_docker_run_named_pipe_output)
self.route('POST', ('test_docker_run_girder_file_to_named_pipe', ),
self.test_docker_run_girder_file_to_named_pipe)
self.route('POST', ('test_docker_run_file_upload_to_item', ),
self.test_docker_run_file_upload_to_item)
self.route('POST', ('test_docker_run_girder_file_to_named_pipe_on_temp_vol', ),
self.test_docker_run_girder_file_to_named_pipe_on_temp_vol)
self.route('POST', ('test_docker_run_mount_idiomatic_volume', ),
self.test_docker_run_mount_idiomatic_volume)
self.route('POST', ('test_docker_run_progress_pipe', ),
self.test_docker_run_progress_pipe)
self.route('POST', ('test_docker_run_girder_file_to_volume', ),
self.test_docker_run_girder_file_to_volume)
self.route('POST', ('input_stream',), self.input_stream)
self.route('POST', ('test_docker_run_transfer_encoding_stream', ),
self.test_docker_run_transfer_encoding_stream)
self.route('POST', ('test_docker_run_temporary_volume_root', ),
self.test_docker_run_temporary_volume_root)
self.route('POST', ('test_docker_run_raises_exception', ),
self.test_docker_run_raises_exception)
self.route('POST', ('test_docker_run_cancel', ),
self.test_docker_run_cancel)
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test basic docker_run.'))
def test_docker_run(self, params):
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['stdio', '-m', 'hello docker!'],
remove_container=True)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test docker run that raises an exception.'))
def test_docker_run_raises_exception(self, params):
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['raise_exception'], remove_container=True)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test mounting a volume.'))
def test_docker_run_mount_volume(self, params):
fixture_dir = params.get('fixtureDir')
filename = 'read.txt'
mount_dir = '/mnt/test'
mount_path = os.path.join(mount_dir, filename)
volumes = {
fixture_dir: {
'bind': mount_dir,
'mode': 'ro'
}
}
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['read', '-p', mount_path],
remove_container=True, volumes=volumes)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test named pipe output.'))
def test_docker_run_named_pipe_output(self, params):
tmp_dir = params.get('tmpDir')
message = params.get('message')
mount_dir = '/mnt/girder_worker/data'
pipe_name = 'output_pipe'
volumes = {
tmp_dir: {
'bind': mount_dir,
'mode': 'rw'
}
}
connect = Connect(NamedOutputPipe(pipe_name, mount_dir, tmp_dir), HostStdOut())
result = docker_run.delay(
TEST_IMAGE, pull_image=True,
container_args=['write', '-p', connect, '-m', message],
remove_container=True, volumes=volumes)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test downloading file using named pipe.'))
def test_docker_run_girder_file_to_named_pipe(self, params):
tmp_dir = params.get('tmpDir')
file_id = params.get('fileId')
mount_dir = '/mnt/girder_worker/data'
pipe_name = 'input_pipe'
volumes = {
tmp_dir: {
'bind': mount_dir,
'mode': 'rw'
}
}
connect = Connect(GirderFileIdToStream(file_id),
NamedInputPipe(pipe_name, mount_dir, tmp_dir))
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['read', '-p', connect],
remove_container=True, volumes=volumes)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test uploading output file to item.'))
def test_docker_run_file_upload_to_item(self, params):
item_id = params.get('itemId')
contents = params.get('contents')
volumepath = VolumePath('test_file')
result = docker_run.delay(
TEST_IMAGE, pull_image=True,
container_args=['write', '-p', volumepath, '-m', contents],
remove_container=True,
girder_result_hooks=[GirderUploadVolumePathToItem(volumepath, item_id)])
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test downloading file using named pipe.'))
def test_docker_run_girder_file_to_named_pipe_on_temp_vol(self, params):
"""
This is a simplified version of test_docker_run_girder_file_to_named_pipe
it uses the TemporaryVolume, rather than having to setup the volumes
'manually', this is the approach we should encourage.
"""
file_id = params.get('fileId')
pipe_name = 'input_pipe'
connect = Connect(GirderFileIdToStream(file_id), NamedInputPipe(pipe_name))
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['read', '-p', connect],
remove_container=True)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test idiomatic volume.'))
def test_docker_run_mount_idiomatic_volume(self, params):
fixture_dir = params.get('fixtureDir')
filename = 'read.txt'
mount_dir = '/mnt/test'
mount_path = os.path.join(mount_dir, filename)
volume = BindMountVolume(fixture_dir, mount_path, 'ro')
volumepath = VolumePath(filename, volume)
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['read', '-p', volumepath],
remove_container=True, volumes=[volume])
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test progress pipe.'))
def test_docker_run_progress_pipe(self, params):
progressions = params.get('progressions')
progress_pipe = ProgressPipe()
result = docker_run.delay(
TEST_IMAGE, pull_image=True,
container_args=['progress', '-p', progress_pipe, '--progressions', progressions],
remove_container=True)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test download to volume.'))
def test_docker_run_girder_file_to_volume(self, params):
file_id = params.get('fileId')
result = docker_run.delay(
TEST_IMAGE, pull_image=True,
container_args=['read_write', '-i', GirderFileIdToVolume(file_id),
'-o', Connect(NamedOutputPipe('out'), HostStdOut())],
remove_container=True)
return result.job
@access.token
@autoDescribeRoute(
Description('Accept transfer encoding request. Used by '
'test_docker_run_transfer_encoding_stream test case.')
.modelParam('itemId', 'The item id',
model=Item, destName='item',
level=AccessType.READ, paramType='query')
.param('delimiter', 'Delimiter to use when writing out chunks.'))
def input_stream(self, item, delimiter):
chunks = six.BytesIO()
for chunk in iterBody(1):
chunks.write(chunk)
chunks.write(delimiter.encode('utf-8'))
chunks.seek(0)
contents = chunks.read()
chunks.seek(0)
Upload().uploadFromFile(
chunks, len(contents), 'chunks', parentType='item', parent=item,
user=getCurrentUser())
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test transfer encoding stream.'))
def test_docker_run_transfer_encoding_stream(self, params):
item_id = params.get('itemId')
file_id = params.get('fileId')
delimiter = params.get('delimiter')
headers = {
'Girder-Token': str(Token().createToken(getCurrentUser())['_id'])
}
url = '%s/%s?itemId=%s&delimiter=%s' % (
getApiUrl(), 'integration_tests/docker/input_stream', item_id, delimiter)
container_args = [
'read_write',
'-i', GirderFileIdToVolume(file_id),
'-o', Connect(NamedOutputPipe('out'),
ChunkedTransferEncodingStream(url, headers))
]
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=container_args,
remove_container=True)
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test setting temporary volume root.'))
def test_docker_run_temporary_volume_root(self, params):
prefix = params.get('prefix')
root = os.path.join(tempfile.gettempdir(), prefix)
# We set the mode to 0o777 because the worker container is
# running as the 'worker' user and needs to be able to have
# read/write access to the TemporaryVolume
volume = TemporaryVolume(host_dir=root, mode=0o777)
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=['print_path', '-p', volume],
remove_container=True, volumes=[volume])
return result.job
@access.token
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Test cancel docker_run.'))
def test_docker_run_cancel(self, params):
mode = params.get('mode')
result = docker_run.delay(
TEST_IMAGE, pull_image=True, container_args=[mode],
remove_container=True)
assert wait_for_status(self.getCurrentUser(), result.job, JobStatus.RUNNING)
result.revoke()
return result.job
class Projects(Resource):
def __init__(self):
super(Projects, self).__init__()
self.resourceName = 'projects'
self.route('POST', (), self.create)
self.route('PATCH', (':id', ), self.update)
self.route('GET', (), self.get_all)
self.route('DELETE', (':id', ), self.delete)
self.route('PUT', (':id', 'share'), self.share)
self.route('POST', (':id', 'simulations'), self.create_simulation)
self.route('GET', (':id', 'simulations'), self.simulations)
self.route('GET', (':id', ), self.get)
self._model = self.model('project', 'hpccloud')
addModel('ProjectProperties', schema.project, 'projects')
@describeRoute(
Description('Create a new project').param(
'body',
'The properies of the project.',
dataType='ProjectProperties',
required=True,
paramType='body'))
@access.user
def create(self, params):
project = getBodyJson()
project = self.model('project',
'hpccloud').create(getCurrentUser(), project)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] = '/projects/%s' % project['_id']
return project
@describeRoute(
Description('Update a project').param(
'id',
'The project to update.',
dataType='string',
required=True,
paramType='path').param('body',
'The properies of the project to update.',
dataType='object',
required=True,
paramType='body'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
def update(self, project, params):
immutable = [
'type', 'steps', 'folderId', 'access', 'userId', '_id', 'created',
'updated'
]
updates = getBodyJson()
for p in updates:
if p in immutable:
raise RestException('\'%s\' is an immutable property' % p, 400)
user = getCurrentUser()
name = updates.get('name')
metadata = updates.get('metadata')
description = updates.get('description')
return self._model.update(user,
project,
name=name,
metadata=metadata,
description=description)
@describeRoute(
Description('Get all projects this user has access to project').param(
'limit',
'Result set size limit.',
dataType='integer',
required=False,
paramType='query').param('offset',
'Offset into result set.',
dataType='integer',
required=False,
paramType='query'))
@access.user
def get_all(self, params):
user = getCurrentUser()
limit, offset, _ = self.getPagingParameters(params)
cursor = self._model.find(limit=limit, offset=offset)
return list(
self._model.filterResultsByPermission(cursor=cursor,
user=user,
level=AccessType.READ))
@describeRoute(
Description('Delete a project').param(
'id',
'The project to delete.',
dataType='string',
required=True,
paramType='path').notes(
'Will clean up any files, items or folders associated with '
'the project.'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
def delete(self, project, params):
user = getCurrentUser()
self._model.delete(user, project)
@describeRoute(
Description('Get a particular project').param('id',
'The project to get.',
dataType='string',
required=True,
paramType='path'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
def get(self, project, params):
return project
addModel('ShareProperties', schema.project['definitions']['share'],
'projects')
@describeRoute(
Description(
'Share a give project with a set of users or groups').param(
'id',
'The project to shared.',
dataType='string',
required=True,
paramType='path'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.WRITE)
def share(self, project, params):
body = getBodyJson()
user = getCurrentUser()
# Validate we have been given a value body
try:
ref_resolver = jsonschema.RefResolver.from_schema(
schema.definitions)
jsonschema.validate(body,
schema.project['definitions']['share'],
resolver=ref_resolver)
except jsonschema.ValidationError as ve:
raise RestException(ve.message, 400)
users = body.get('users', [])
groups = body.get('groups', [])
return self._model.share(user, project, users, groups)
addModel('SimProperties', schema.simulation, 'projects')
@describeRoute(
Description('Create a simulation associated with a project.').param(
'id',
'The project the simulation will be created in.',
dataType='string',
required=True,
paramType='path').param('body',
'The properties of the simulation.',
dataType='SimProperties',
required=True,
paramType='body'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
def create_simulation(self, project, params):
simulation = getBodyJson()
user = getCurrentUser()
simulation = self.model('simulation',
'hpccloud').create(user, project, simulation)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] = '/simulations/%s' \
% simulation['_id']
return simulation
@describeRoute(
Description(
'List all the simulations associated with a project.').param(
'id',
'The project',
dataType='string',
required=True,
paramType='path').param('limit',
'Result set size limit.',
dataType='integer',
required=False,
paramType='query').param(
'offset',
'Offset into result set.',
dataType='integer',
required=False,
paramType='query'))
@access.user
@loadmodel(model='project', plugin='hpccloud', level=AccessType.READ)
def simulations(self, project, params):
user = getCurrentUser()
limit, offset, _ = self.getPagingParameters(params)
return self.model('project',
'hpccloud').simulations(user, project, limit, offset)
class MockSlicerCLIWebResource(Resource):
"""
This creates a mocked version of the ``/HistomicsTK/HistomicsTK/docker_image``
endpoint so we can test generation of the analysis panel on the client without
relying on girder_worker + docker.
"""
def __init__(self):
super(MockSlicerCLIWebResource, self).__init__()
self.route('GET', ('docker_image', ), self.dockerImage)
self.route('GET', ('test_analysis_detection', 'xml'),
self.testAnalysisXmlDetection)
self.route('GET', ('test_analysis_features', 'xml'),
self.testAnalysisXmlFeatures)
self.route('POST', ('test_analysis_detection', 'run'),
self.testAnalysisRun)
self.route('POST', ('test_analysis_features', 'run'),
self.testAnalysisRun)
@access.public
@describeRoute(Description('Mock the docker_image endpoint.'))
def dockerImage(self, params):
"""
Return a single CLI referencing mocked out /xmlspec and /run endpoints.
"""
return {
'huirchive/histomicstk': {
'latest': {
'ComputeNucleiFeatures': {
'run': 'mock_resource/test_analysis_features/run',
'type': 'python',
'xmlspec': 'mock_resource/test_analysis_features/xml'
},
'NucleiDetection': {
'run': 'mock_resource/test_analysis_detection/run',
'type': 'python',
'xmlspec': 'mock_resource/test_analysis_detection/xml'
}
}
}
}
@access.public
@describeRoute(Description('Mock an analysis description route.'))
def testAnalysisXmlDetection(self, params):
"""Return the nuclei detection XML spec as a test case."""
xml_file = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'test_analysis_detection.xml'))
with open(xml_file) as f:
xml = f.read()
setResponseHeader('Content-Type', 'application/xml')
setRawResponse()
return xml
@access.public
@describeRoute(Description('Mock an analysis description route.'))
def testAnalysisXmlFeatures(self, params):
"""Return the nuclei feature classification XML spec as a test case."""
xml_file = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'test_analysis_features.xml'))
with open(xml_file) as f:
xml = f.read()
setResponseHeader('Content-Type', 'application/xml')
setRawResponse()
return xml
@access.public
@describeRoute(Description('Mock an analysis run route.'))
def testAnalysisRun(self, params):
"""
Mock out the CLI execution endpoint.
For now, this is a no-op, but we should add some logic to generate an annotation
output and job status events to simulate a real execution of the CLI.
"""
return {'_id': 'jobid'}
class Calculation(Resource):
output_formats = ['cml', 'xyz', 'inchikey', 'sdf']
input_formats = ['cml', 'xyz', 'pdb']
def __init__(self):
super(Calculation, self).__init__()
self.resourceName = 'calculations'
self.route('POST', (), self.create_calc)
self.route('PUT', (':id', ), self.ingest_calc)
self.route('DELETE', (':id', ), self.delete)
self.route('GET', (), self.find_calc)
self.route('GET', ('types', ), self.find_calc_types)
self.route('GET', (':id', 'vibrationalmodes'),
self.get_calc_vibrational_modes)
self.route('GET', (':id', 'vibrationalmodes', ':mode'),
self.get_calc_vibrational_mode)
self.route('GET', (':id', 'sdf'), self.get_calc_sdf)
self.route('GET', (':id', 'cjson'), self.get_calc_cjson)
self.route('GET', (':id', 'xyz'), self.get_calc_xyz)
self.route('GET', (':id', 'cube', ':mo'), self.get_calc_cube)
self.route('GET', (':id', ), self.find_id)
self.route('PUT', (':id', 'properties'), self.update_properties)
self.route('PATCH', (':id', 'notebooks'), self.add_notebooks)
self._model = ModelImporter.model('calculation', 'molecules')
self._cube_model = ModelImporter.model('cubecache', 'molecules')
@access.public
def get_calc_vibrational_modes(self, id, params):
# TODO: remove 'cjson' once girder issue #2883 is resolved
fields = [
'cjson', 'cjson.vibrations.modes', 'cjson.vibrations.intensities',
'cjson.vibrations.frequencies', 'access'
]
calc = self._model.load(id,
fields=fields,
user=getCurrentUser(),
level=AccessType.READ)
del calc['access']
if 'cjson' in calc and 'vibrations' in calc['cjson']:
return calc['cjson']['vibrations']
else:
return {'modes': [], 'intensities': [], 'frequencies': []}
get_calc_vibrational_modes.description = (Description(
'Get the vibrational modes associated with a calculation').param(
'id',
'The id of the calculation to get the modes from.',
dataType='string',
required=True,
paramType='path'))
@access.public
def get_calc_vibrational_mode(self, id, mode, params):
try:
mode = int(mode)
except ValueError:
raise ValidationException('mode number be an integer', 'mode')
# TODO: remove 'cjson' once girder issue #2883 is resolved
fields = ['cjson', 'cjson.vibrations.modes', 'access']
calc = self._model.load(id,
fields=fields,
user=getCurrentUser(),
level=AccessType.READ)
vibrational_modes = calc['cjson']['vibrations']
#frames = vibrational_modes.get('modeFrames')
modes = vibrational_modes.get('modes', [])
index = modes.index(mode)
if index < 0:
raise RestException('No such vibrational mode', 400)
# Now select the modeFrames directly this seems to be more efficient
# than iterating in Python
query = {'_id': calc['_id']}
projection = {
'cjson.vibrations.frequencies': {
'$slice': [index, 1]
},
'cjson.vibrations.intensities': {
'$slice': [index, 1]
},
'cjson.vibrations.eigenVectors': {
'$slice': [index, 1]
},
'cjson.vibrations.modes': {
'$slice': [index, 1]
}
}
mode = self._model.findOne(query, fields=projection)
return mode['cjson']['vibrations']
get_calc_vibrational_mode.description = (Description(
'Get a vibrational mode associated with a calculation').param(
'id',
'The id of the calculation that the mode is associated with.',
dataType='string',
required=True,
paramType='path').param(
'mode',
'The index of the vibrational model to get.',
dataType='string',
required=True,
paramType='path'))
@access.public
@loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
def get_calc_sdf(self, calculation, params):
def stream():
cherrypy.response.headers['Content-Type'] = 'chemical/x-mdl-sdfile'
yield calculation['sdf']
return stream
get_calc_sdf.description = (Description(
'Get the molecular structure of a give calculation in SDF format'
).param('id',
'The id of the calculation to return the structure for.',
dataType='string',
required=True,
paramType='path'))
@access.public
@loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
def get_calc_cjson(self, calculation, params):
return calculation['cjson']
get_calc_cjson.description = (Description(
'Get the molecular structure of a give calculation in CJSON format'
).param('id',
'The id of the calculation to return the structure for.',
dataType='string',
required=True,
paramType='path'))
@access.public
@loadmodel(model='calculation', plugin='molecules', level=AccessType.READ)
def get_calc_xyz(self, calculation, params):
data = json.dumps(calculation['cjson'])
data = avogadro.convert_str(data, 'cjson', 'xyz')
def stream():
cherrypy.response.headers['Content-Type'] = Molecule.mime_types[
'xyz']
yield data
return stream
get_calc_xyz.description = (Description(
'Get the molecular structure of a give calculation in XYZ format'
).param('id',
'The id of the calculation to return the structure for.',
dataType='string',
required=True,
paramType='path'))
@access.public
def get_calc_cube(self, id, mo, params):
orig_mo = mo
try:
mo = int(mo)
except ValueError:
# Check for h**o lumo
mo = mo.lower()
if mo in ['h**o', 'lumo']:
cal = self._model.load(id, force=True)
# Electron count might be saved in several places...
path_expressions = [
'cjson.orbitals.electronCount',
'cjson.basisSet.electronCount', 'properties.electronCount'
]
matches = []
for expr in path_expressions:
matches.extend(parse(expr).find(cal))
if len(matches) > 0:
electron_count = matches[0].value
else:
raise RestException('Unable to access electronCount', 400)
# The index of the first orbital is 0, so h**o needs to be
# electron_count // 2 - 1
if mo == 'h**o':
mo = int(electron_count / 2) - 1
elif mo == 'lumo':
mo = int(electron_count / 2)
else:
raise ValidationException(
'mo number be an integer or \'h**o\'/\'lumo\'', 'mode')
cached = self._cube_model.find_mo(id, mo)
# If we have a cached cube file use that.
if cached:
return cached['cjson']
fields = ['cjson', 'access', 'fileId']
# Ignoring access control on file/data for now, all public.
calc = self._model.load(id, fields=fields, force=True)
# This is where the cube gets calculated, should be cached in future.
if ('async' in params) and (params['async']):
async_requests.schedule_orbital_gen(calc['cjson'], mo, id, orig_mo,
self.getCurrentUser())
calc['cjson']['cube'] = {
'dimensions': [0, 0, 0],
'scalars': [],
'origin': [0, 0, 0],
'spacing': [1, 1, 1]
}
return calc['cjson']
else:
cjson = avogadro.calculate_mo(calc['cjson'], mo)
# Remove the vibrational mode data from the cube - big, not needed here.
if 'vibrations' in cjson:
del cjson['vibrations']
# Cache this cube for the next time, they can take a while to generate.
self._cube_model.create(id, mo, cjson)
return cjson
get_calc_cube.description = (Description(
'Get the cube for the supplied MO of the calculation in CJSON format'
).param('id',
'The id of the calculation to return the structure for.',
dataType='string',
required=True,
paramType='path').param(
'mo',
'The molecular orbital to get the cube for.',
dataType='string',
required=True,
paramType='path'))
@access.user(scope=TokenScope.DATA_WRITE)
def create_calc(self, params):
body = getBodyJson()
if 'cjson' not in body and ('fileId' not in body
or 'format' not in body):
raise RestException('Either cjson or fileId is required.')
user = getCurrentUser()
cjson = body.get('cjson')
props = body.get('properties', {})
molecule_id = body.get('moleculeId', None)
geometry_id = body.get('geometryId', None)
public = body.get('public', True)
notebooks = body.get('notebooks', [])
image = body.get('image')
input_parameters = body.get('input', {}).get('parameters')
if input_parameters is None:
input_parameters = body.get('inputParameters', {})
file_id = None
file_format = body.get('format', 'cjson')
if 'fileId' in body:
file = File().load(body['fileId'], user=getCurrentUser())
file_id = file['_id']
cjson = self._file_to_cjson(file, file_format)
if molecule_id is None:
mol = create_molecule(json.dumps(cjson),
'cjson',
user,
public,
parameters=input_parameters)
molecule_id = mol['_id']
calc = CalculationModel().create_cjson(
user,
cjson,
props,
molecule_id,
geometry_id=geometry_id,
image=image,
input_parameters=input_parameters,
file_id=file_id,
notebooks=notebooks,
public=public)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] \
= '/calculations/%s' % (str(calc['_id']))
return CalculationModel().filter(calc, user)
# Try and reuse schema for documentation, this only partially works!
calc_schema = CalculationModel.schema.copy()
calc_schema['id'] = 'CalculationData'
addModel('Calculation', 'CalculationData', calc_schema)
create_calc.description = (Description(
'Get the molecular structure of a give calculation in SDF format').
param('body',
'The calculation data',
dataType='CalculationData',
required=True,
paramType='body'))
def _file_to_cjson(self, file, file_format):
readers = {'cjson': oc.CjsonReader}
if file_format not in readers:
raise Exception('Unknown file format %s' % file_format)
reader = readers[file_format]
with File().open(file) as f:
calc_data = f.read().decode()
# SpooledTemporaryFile doesn't implement next(),
# workaround in case any reader needs it
tempfile.SpooledTemporaryFile.__next__ = lambda self: self.__iter__(
).__next__()
with tempfile.SpooledTemporaryFile(mode='w+',
max_size=10 * 1024 * 1024) as tf:
tf.write(calc_data)
tf.seek(0)
cjson = reader(tf).read()
return cjson
@access.user(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Update pending calculation with results.').modelParam(
'id',
'The calculation id',
model=CalculationModel,
destName='calculation',
level=AccessType.WRITE,
paramType='path').
param(
'detectBonds',
'Automatically detect bonds if they are not already present in the ingested molecule',
required=False,
dataType='boolean',
default=False).jsonParam('body',
'The calculation details',
required=True,
paramType='body'))
def ingest_calc(self, calculation, body, detectBonds=None):
self.requireParams(['fileId', 'format'], body)
file = File().load(body['fileId'], user=getCurrentUser())
cjson = self._file_to_cjson(file, body['format'])
calc_props = calculation['properties']
# The calculation is no longer pending
if 'pending' in calc_props:
del calc_props['pending']
# Add bonds if they were not there already
if detectBonds is None:
detectBonds = False
bonds = cjson.get('bonds')
if bonds is None and detectBonds:
new_cjson = openbabel.autodetect_bonds(cjson)
if new_cjson.get('bonds') is not None:
cjson['bonds'] = new_cjson['bonds']
calculation['properties'] = calc_props
calculation['cjson'] = cjson
calculation['fileId'] = file['_id']
image = body.get('image')
if image is not None:
calculation['image'] = image
code = body.get('code')
if code is not None:
calculation['code'] = code
scratch_folder_id = body.get('scratchFolderId')
if scratch_folder_id is not None:
calculation['scratchFolderId'] = scratch_folder_id
# If this was a geometry optimization, create a geometry from it
task = parse('input.parameters.task').find(calculation)
if task and task[0].value == 'optimize':
moleculeId = calculation.get('moleculeId')
provenanceType = 'calculation'
provenanceId = calculation.get('_id')
# The cjson will be whitelisted
geometry = GeometryModel().create(getCurrentUser(), moleculeId,
cjson, provenanceType,
provenanceId)
calculation['optimizedGeometryId'] = geometry.get('_id')
return CalculationModel().save(calculation)
@access.public
@autoDescribeRoute(
Description('Search for particular calculation').param(
'moleculeId',
'The molecule ID linked to this calculation',
required=False).param(
'geometryId',
'The geometry ID linked to this calculation',
required=False).param(
'imageName',
'The name of the Docker image that run this calculation',
required=False).
param(
'inputParameters',
'JSON string of the input parameters. May be in percent encoding.',
required=False).param('inputGeometryHash',
'The hash of the input geometry.',
required=False).param(
'name',
'The name of the molecule',
paramType='query',
required=False).param(
'inchi',
'The InChI of the molecule',
paramType='query',
required=False).
param('inchikey',
'The InChI key of the molecule',
paramType='query',
required=False).param(
'smiles',
'The SMILES of the molecule',
paramType='query',
required=False).param(
'formula',
'The formula (using the "Hill Order") to search for',
paramType='query',
required=False).param(
'creatorId',
'The id of the user that created the calculation',
required=False).pagingParams(
defaultSort='_id',
defaultSortDir=SortDir.DESCENDING,
defaultLimit=25))
def find_calc(self,
moleculeId=None,
geometryId=None,
imageName=None,
inputParameters=None,
inputGeometryHash=None,
name=None,
inchi=None,
inchikey=None,
smiles=None,
formula=None,
creatorId=None,
pending=None,
limit=None,
offset=None,
sort=None):
return CalculationModel().findcal(
molecule_id=moleculeId,
geometry_id=geometryId,
image_name=imageName,
input_parameters=inputParameters,
input_geometry_hash=inputGeometryHash,
name=name,
inchi=inchi,
inchikey=inchikey,
smiles=smiles,
formula=formula,
creator_id=creatorId,
pending=pending,
limit=limit,
offset=offset,
sort=sort,
user=getCurrentUser())
@access.public
def find_id(self, id, params):
user = getCurrentUser()
cal = self._model.load(id, level=AccessType.READ, user=user)
if not cal:
raise RestException('Calculation not found.', code=404)
return cal
find_id.description = (Description('Get the calculation by id').param(
'id',
'The id of calculation.',
dataType='string',
required=True,
paramType='path'))
@access.user(scope=TokenScope.DATA_WRITE)
def delete(self, id, params):
user = getCurrentUser()
cal = self._model.load(id, level=AccessType.READ, user=user)
if not cal:
raise RestException('Calculation not found.', code=404)
return self._model.remove(cal, user)
delete.description = (Description('Delete a calculation by id.').param(
'id',
'The id of calculatino.',
dataType='string',
required=True,
paramType='path'))
@access.public
def find_calc_types(self, params):
fields = ['access', 'properties.calculationTypes']
query = {}
if 'moleculeId' in params:
query['moleculeId'] = ObjectId(params['moleculeId'])
calcs = self._model.find(query, fields=fields)
allTypes = []
for types in calcs:
calc_types = parse('properties.calculationTypes').find(types)
if calc_types:
calc_types = calc_types[0].value
allTypes.extend(calc_types)
typeSet = set(allTypes)
return list(typeSet)
find_calc_types.description = (Description(
'Get the calculation types available for the molecule').param(
'moleculeId',
'The id of the molecule we are finding types for.',
dataType='string',
required=True,
paramType='query'))
@access.user(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Update the calculation properties.').notes(
'Override the exist properties').modelParam(
'id',
'The ID of the calculation.',
model='calculation',
plugin='molecules',
level=AccessType.ADMIN).param(
'body', 'The new set of properties',
paramType='body').errorResponse(
'ID was invalid.').errorResponse(
'Write access was denied for the calculation.',
403))
def update_properties(self, calculation, params):
props = getBodyJson()
calculation['properties'] = props
calculation = self._model.save(calculation)
return calculation
@access.user(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Add notebooks ( file ids ) to molecule.').modelParam(
'id',
'The calculation id',
model=CalculationModel,
destName='calculation',
force=True,
paramType='path').jsonParam('notebooks',
'List of notebooks',
required=True,
paramType='body'))
def add_notebooks(self, calculation, notebooks):
notebooks = notebooks.get('notebooks')
if notebooks is not None:
CalculationModel().add_notebooks(calculation, notebooks)
def genRESTEndPointsForSlicerCLIsInDocker(info, restResource, dockerImages):
"""Generates REST end points for slicer CLIs placed in subdirectories of a
given root directory and attaches them to a REST resource with the given
name.
For each CLI, it creates:
* a GET Route (<apiURL>/`restResourceName`/<cliRelativePath>/xmlspec)
that returns the xml spec of the CLI
* a POST Route (<apiURL>/`restResourceName`/<cliRelativePath>/run)
that runs the CLI
It also creates a GET route (<apiURL>/`restResourceName`) that returns a
list of relative routes to all CLIs attached to the generated REST resource
Parameters
----------
info
restResource : str or girder.api.rest.Resource
REST resource to which the end-points should be attached
dockerImages : a list of docker image names
"""
dockerImages
# validate restResource argument
if not isinstance(restResource, (str, Resource)):
raise Exception('restResource must either be a string or '
'an object of girder.api.rest.Resource')
# validate dockerImages arguments
if not isinstance(dockerImages, (str, list)):
raise Exception('dockerImages must either be a single docker image '
'string or a list of docker image strings')
if isinstance(dockerImages, list):
for img in dockerImages:
if not isinstance(img, str):
raise Exception('dockerImages must either be a single '
'docker image string or a list of docker '
'image strings')
else:
dockerImages = [dockerImages]
# create REST resource if given a name
if isinstance(restResource, str):
restResource = type(restResource, (Resource, ),
{'resourceName': restResource})()
restResourceName = type(restResource).__name__
# Add REST routes for slicer CLIs in each docker image
cliList = []
for dimg in dockerImages:
# check if the docker image exists
getDockerImage(dimg, True)
# get CLI list
cliListSpec = getDockerImageCLIList(dimg)
cliListSpec = json.loads(cliListSpec)
# Add REST end-point for each CLI
for cliRelPath in cliListSpec.keys():
cliXML = getDockerImageCLIXMLSpec(dimg, cliRelPath)
# create a POST REST route that runs the CLI
try:
cliRunHandler = genHandlerToRunDockerCLI(
dimg, cliRelPath, cliXML, restResource)
except Exception:
logger.execption('Failed to create REST endpoints for %s',
cliRelPath)
continue
cliSuffix = os.path.normpath(cliRelPath).replace(os.sep, '_')
cliRunHandlerName = 'run_' + cliSuffix
setattr(restResource, cliRunHandlerName, cliRunHandler)
restResource.route('POST', (cliRelPath, 'run'),
getattr(restResource, cliRunHandlerName))
# create GET REST route that returns the xml of the CLI
try:
cliGetXMLSpecHandler = genHandlerToGetDockerCLIXmlSpec(
cliRelPath, cliXML, restResource)
except Exception:
logger.exception('Failed to create REST endpoints for %s',
cliRelPath)
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error('%r' % [exc_type, fname, exc_tb.tb_lineno])
continue
cliGetXMLSpecHandlerName = 'get_xml_' + cliSuffix
setattr(restResource, cliGetXMLSpecHandlerName,
cliGetXMLSpecHandler)
restResource.route('GET', (
cliRelPath,
'xmlspec',
), getattr(restResource, cliGetXMLSpecHandlerName))
cliList.append(cliRelPath)
# create GET route that returns a list of relative routes to all CLIs
@boundHandler(restResource)
@access.user
@describeRoute(Description('Get list of relative routes to all CLIs'))
def getCLIListHandler(self, *args, **kwargs):
return cliList
getCLIListHandlerName = 'get_cli_list'
setattr(restResource, getCLIListHandlerName, getCLIListHandler)
restResource.route('GET', (), getattr(restResource, getCLIListHandlerName))
# expose the generated REST resource via apiRoot
setattr(info['apiRoot'], restResourceName, restResource)
# return restResource
return restResource
class HistomicsTKResource(DockerResource):
def __init__(self, name, *args, **kwargs):
super(HistomicsTKResource, self).__init__(name, *args, **kwargs)
self.route('GET', ('settings',), self.getPublicSettings)
self.route('GET', ('analysis', 'access'), self.getAnalysisAccess)
def _accessList(self):
access = Setting().get(PluginSettings.HISTOMICSTK_ANALYSIS_ACCESS) or {}
acList = {
'users': [{'id': x, 'level': AccessType.READ}
for x in access.get('users', [])],
'groups': [{'id': x, 'level': AccessType.READ}
for x in access.get('groups', [])],
'public': access.get('public', True),
}
for user in acList['users'][:]:
userDoc = User().load(
user['id'], force=True,
fields=['firstName', 'lastName', 'login'])
if userDoc is None:
acList['users'].remove(user)
else:
user['login'] = userDoc['login']
user['name'] = ' '.join((userDoc['firstName'], userDoc['lastName']))
for grp in acList['groups'][:]:
grpDoc = Group().load(
grp['id'], force=True, fields=['name', 'description'])
if grpDoc is None:
acList['groups'].remove(grp)
else:
grp['name'] = grpDoc['name']
grp['description'] = grpDoc['description']
return acList
@access.public
@describeRoute(
Description('List docker images and their CLIs')
)
def getDockerImages(self, *args, **kwargs):
user = self.getCurrentUser()
if not user:
return {}
result = super(HistomicsTKResource, self).getDockerImages(*args, **kwargs)
acList = self._accessList()
# Use the User().hasAccess to test for access via a synthetic document
if not User().hasAccess({'access': acList, 'public': acList['public']}, user):
return {}
return result
@describeRoute(
Description('Get public settings for HistomicsTK.')
)
@access.public
def getPublicSettings(self, params):
keys = [
PluginSettings.HISTOMICSTK_DEFAULT_DRAW_STYLES,
]
return {k: self.model('setting').get(k) for k in keys}
@describeRoute(
Description('Get the access list for analyses.')
)
@access.admin
def getAnalysisAccess(self, params):
return self._accessList()
class Thumbnail(Resource):
def __init__(self):
super(Thumbnail, self).__init__()
self.resourceName = 'thumbnail'
self.route('POST', (), self.createThumbnail)
@access.user
@loadmodel(map={'fileId': 'file'}, model='file', level=AccessType.READ)
@filtermodel(model='job', plugin='jobs')
@describeRoute(
Description('Create a new thumbnail from an existing image file.')
.notes('Setting a width or height parameter of 0 will preserve the '
'original aspect ratio.')
.param('fileId', 'The ID of the source file.')
.param('width', 'The desired width.', required=False, dataType='int')
.param('height', 'The desired height.', required=False, dataType='int')
.param('crop', 'Whether to crop the image to preserve aspect ratio. '
'Only used if both width and height parameters are nonzero.',
dataType='boolean', required=False, default=True)
.param('attachToId', 'The lifecycle of this thumbnail is bound to the '
'resource specified by this ID.')
.param('attachToType', 'The type of resource to which this thumbnail is'
' attached.', enum=['folder', 'user', 'collection', 'item'])
.errorResponse()
.errorResponse(('Write access was denied on the attach destination.',
'Read access was denied on the file.'), 403)
)
def createThumbnail(self, file, params):
self.requireParams(('attachToId', 'attachToType'), params)
user = self.getCurrentUser()
width = params.get('width')
height = params.get('height')
if params['attachToType'] not in (
'item', 'collection', 'user', 'folder'):
raise RestException('You can only attach thumbnails to users, '
'folders, collections, or items.')
self.model(params['attachToType']).load(
params['attachToId'], user=user, level=AccessType.WRITE, exc=True)
width = max(int(params.get('width', 0)), 0)
height = max(int(params.get('height', 0)), 0)
if not width and not height:
raise RestException(
'You must specify a valid width, height, or both.')
kwargs = {
'width': width,
'height': height,
'fileId': str(file['_id']),
'crop': self.boolParam('crop', params, default=True),
'attachToType': params['attachToType'],
'attachToId': params['attachToId']
}
job = self.model('job', 'jobs').createLocalJob(
title='Generate thumbnail for %s' % file['name'], user=user,
type='thumbnails.create', public=False, kwargs=kwargs,
module='girder.plugins.thumbnails.worker')
self.model('job', 'jobs').scheduleJob(job)
return job
class Volume(BaseResource):
def __init__(self):
super(Volume, self).__init__()
self.resourceName = 'volumes'
self.route('POST', (), self.create)
self.route('GET', (':id', ), self.get)
self.route('PATCH', (':id', ), self.patch)
self.route('GET', (), self.find)
self.route('GET', (':id', 'status'), self.get_status)
self.route('POST', (':id', 'log'), self.append_to_log)
self.route('GET', (':id', 'log'), self.log)
self.route('PUT', (':id', 'clusters', ':clusterId', 'attach'),
self.attach)
self.route('PUT',
(':id', 'clusters', ':clusterId', 'attach', 'complete'),
self.attach_complete)
self.route('PUT', (':id', 'detach'), self.detach)
self.route('PUT', (':id', 'detach', 'complete'), self.detach_complete)
self.route('DELETE', (':id', ), self.delete)
self.route('PUT', (':id', 'delete', 'complete'), self.delete_complete)
self._model = self.model('volume', 'cumulus')
def _create_ebs(self, body, zone):
user = getCurrentUser()
name = body['name']
size = body['size']
fs = body.get('fs', None)
profileId = body['profileId']
return self._model.create_ebs(user, profileId, name, zone, size, fs)
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.WRITE)
def patch(self, volume, params):
body = getBodyJson()
if not volume:
raise RestException('Volume not found.', code=404)
if 'ec2' in body:
if 'ec2' not in volume:
volume['ec2'] = {}
volume['ec2'].update(body['ec2'])
mutable = ['status', 'msg', 'path']
for k in mutable:
if k in body:
volume[k] = body[k]
user = getCurrentUser()
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
patch.description = (Description('Patch a volume').param(
'id', 'The volume id.', paramType='path',
required=True).param('body',
'The properties to use to create the volume.',
required=True,
paramType='body'))
@access.user
def create(self, params):
body = getBodyJson()
self.requireParams(['name', 'type', 'size', 'profileId'], body)
if not VolumeType.is_valid_type(body['type']):
raise RestException('Invalid volume type.', code=400)
profile_id = parse('profileId').find(body)
if not profile_id:
raise RestException('A profile id must be provided', 400)
profile_id = profile_id[0].value
profile, secret_key = _get_profile(profile_id)
if not profile:
raise RestException('Invalid profile', 400)
if 'zone' in body:
zone = body['zone']
else:
zone = profile['availabilityZone']
volume = self._create_ebs(body, zone)
cherrypy.response.status = 201
cherrypy.response.headers['Location'] = '/volumes/%s' % volume['_id']
return self._model.filter(volume, getCurrentUser())
addModel(
'VolumeParameters', {
'id': 'VolumeParameters',
'required': ['name', 'config', 'type', 'zone', 'size'],
'properties': {
'name': {
'type': 'string',
'description': 'The name to give the cluster.'
},
'profileId': {
'type': 'string',
'description': 'Id of profile to use'
},
'type': {
'type':
'string',
'description':
'The type of volume to create ( currently '
'only esb )'
},
'zone': {
'type': 'string',
'description': 'The availability region'
},
'size': {
'type': 'integer',
'description': 'The size of the volume to create'
}
},
}, 'volumes')
create.description = (Description('Create a volume').param(
'body',
'The properties to use to create the volume.',
dataType='VolumeParameters',
required=True,
paramType='body'))
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.READ)
def get(self, volume, params):
return self._model.filter(volume, getCurrentUser())
get.description = (Description('Get a volume').param('id',
'The volume id.',
paramType='path',
required=True))
@access.user
def find(self, params):
user = getCurrentUser()
query = {}
if 'clusterId' in params:
query['clusterId'] = ObjectId(params['clusterId'])
limit = params.get('limit', 50)
volumes = self._model.find(query=query)
volumes = list(volumes)
volumes = self._model \
.filterResultsByPermission(volumes, user, AccessType.ADMIN,
limit=int(limit))
return [self._model.filter(volume, user) for volume in volumes]
find.description = (Description('Search for volumes').param(
'limit',
'The max number of volumes to return',
paramType='query',
required=False,
default=50))
@access.user
@loadmodel(map={'clusterId': 'cluster'},
model='cluster',
plugin='cumulus',
level=AccessType.ADMIN)
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def attach_complete(self, volume, cluster, params):
user = getCurrentUser()
path = params.get('path', None)
# Is path being passed in as apart of the body json?
if path is None:
path = getBodyJson().get('path', None)
if path is not None:
cluster.setdefault('volumes', [])
cluster['volumes'].append(volume['_id'])
cluster['volumes'] = list(set(cluster['volumes']))
volume['status'] = VolumeState.INUSE
volume['path'] = path
# TODO: removing msg should be refactored into
# a general purpose 'update_status' function
# on the volume model. This way msg only referes
# to the current status.
try:
del volume['msg']
except KeyError:
pass
# Add cluster id to volume
volume['clusterId'] = cluster['_id']
self.model('cluster', 'cumulus').save(cluster)
self._model.update_volume(user, volume)
else:
volume['status'] = VolumeState.ERROR
volume['msg'] = 'Volume path was not communicated on complete'
self._model.update_volume(user, volume)
attach_complete.description = None
@access.user
@loadmodel(map={'clusterId': 'cluster'},
model='cluster',
plugin='cumulus',
level=AccessType.ADMIN)
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def attach(self, volume, cluster, params):
body = getBodyJson()
self.requireParams(['path'], body)
path = body['path']
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
# If volume exists it needs to be available to be attached. If
# it doesn't exist it will be created as part of the attach
# playbook.
if aws_volume is not None and \
aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'This volume is not available to attach '
'to a cluster', 400)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!', 400)
cluster = self.model('cluster', 'cumulus').filter(cluster,
getCurrentUser(),
passphrase=False)
cumulus.ansible.tasks.volume.attach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, getCurrentUser()), path,
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.ATTACHING
volume = self._model.update_volume(getCurrentUser(), volume)
return self._model.filter(volume, getCurrentUser())
addModel(
'AttachParameters', {
'id': 'AttachParameters',
'required': ['path'],
'properties': {
'path': {
'type': 'string',
'description': 'The path to mount the volume'
}
}
}, 'volumes')
attach.description = (Description('Attach a volume to a cluster').param(
'id',
'The id of the volume to attach',
required=True,
paramType='path').param('clusterId',
'The cluster to attach the volume to.',
required=True,
paramType='path').param(
'body',
'The properties to template on submit.',
dataType='AttachParameters',
paramType='body'))
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def detach(self, volume, params):
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume is None or aws_volume['state'] != VolumeState.INUSE:
raise RestException('This volume is not attached '
'to a cluster', 400)
if 'clusterId' not in volume:
raise RestException('clusterId is not set on this volume!', 400)
try:
volume['path']
except KeyError:
raise RestException('path is not set on this volume!', 400)
cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
user=getCurrentUser(),
level=AccessType.ADMIN)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!', 400)
user = getCurrentUser()
cluster = self.model('cluster', 'cumulus').filter(cluster,
user,
passphrase=False)
cumulus.ansible.tasks.volume.detach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, user),
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.DETACHING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
detach.description = (Description('Detach a volume from a cluster').param(
'id', 'The id of the attached volume', required=True,
paramType='path'))
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def detach_complete(self, volume, params):
# First remove from cluster
user = getCurrentUser()
cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
user=user,
level=AccessType.ADMIN)
cluster.setdefault('volumes', []).remove(volume['_id'])
del volume['clusterId']
for attr in ['path', 'msg']:
try:
del volume[attr]
except KeyError:
pass
volume['status'] = VolumeState.AVAILABLE
self.model('cluster', 'cumulus').save(cluster)
self._model.save(volume)
send_status_notification('volume', volume)
detach_complete.description = None
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def delete(self, volume, params):
if 'clusterId' in volume:
raise RestException('Unable to delete attached volume')
# If the volume is in state created and it has no ec2 volume id
# associated with it, we should be able to just delete it
if volume['status'] in (VolumeState.CREATED, VolumeState.ERROR):
if 'id' in volume['ec2'] and volume['ec2']['id'] is not None:
raise RestException('Unable to delete volume, it is '
'associated with an ec2 volume %s' %
volume['ec2']['id'])
self._model.remove(volume)
return None
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
# Call EC2 to delete volume
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'Volume must be in an "%s" status to be deleted' %
VolumeState.AVAILABLE, 400)
user = getCurrentUser()
cumulus.ansible.tasks.volume.delete_volume\
.delay(profile, self._model.filter(volume, user),
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.DELETING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
delete.description = (Description('Delete a volume').param(
'id', 'The volume id.', paramType='path', required=True))
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def delete_complete(self, volume, params):
self._model.remove(volume)
delete_complete.description = None
@access.user
@loadmodel(model='volume', plugin='cumulus', level=AccessType.ADMIN)
def get_status(self, volume, params):
return {'status': volume['status']}
get_status.description = (Description('Get the status of a volume').param(
'id', 'The volume id.', paramType='path', required=True))
@access.user
def append_to_log(self, id, params):
user = getCurrentUser()
if not self._model.load(id, user=user, level=AccessType.ADMIN):
raise RestException('Volume not found.', code=404)
return self._model.append_to_log(user, id, getBodyJson())
append_to_log.description = None
@access.user
def log(self, id, params):
user = getCurrentUser()
offset = 0
if 'offset' in params:
offset = int(params['offset'])
if not self._model.load(id, user=user, level=AccessType.READ):
raise RestException('Volume not found.', code=404)
log_records = self._model.log_records(user, id, offset)
return {'log': log_records}
log.description = (Description('Get log entries for volume').param(
'id', 'The volume to get log entries for.',
paramType='path').param('offset',
'The offset to start getting entries at.',
required=False,
paramType='query'))
class GeospatialItem(Resource):
"""
Geospatial methods added to the API endpoint for items.
"""
@access.user
@filtermodel(Item)
@autoDescribeRoute(
Description(
'Create new items from a GeoJSON feature or feature collection.'
).modelParam('folderId',
'The ID of the parent folder.',
model=Folder,
level=AccessType.WRITE,
paramType='formData').jsonParam(
'geoJSON',
'A GeoJSON object containing the features or feature'
' collection to add.').errorResponse().errorResponse(
'Invalid GeoJSON was passed in request body.').
errorResponse(
'GeoJSON feature or feature collection was not passed in'
' request body.').errorResponse(
"GeoJSON feature did not contain a property named"
" 'name'.").errorResponse(
'Property name was invalid.').errorResponse(
'Write access was denied on the parent folder.', 403).
notes("All GeoJSON features must contain a property named 'name' from"
" which the name of each created item is taken."))
def create(self, folder, geoJSON):
try:
GeoJSON.to_instance(geoJSON, strict=True)
except ValueError:
raise RestException('Invalid GeoJSON passed in request body.')
if geoJSON['type'] == 'Feature':
features = [geoJSON]
elif geoJSON['type'] == 'FeatureCollection':
features = geoJSON['features']
else:
raise RestException(
'GeoJSON feature or feature collection must be '
'passed in request body.')
data = []
for feature in features:
properties = feature['properties']
if 'name' not in properties:
raise RestException("All GeoJSON features must contain a"
" property named 'name'.")
name = properties['name']
del properties['name']
if 'description' in properties:
description = properties['description']
del properties['description']
else:
description = ''
for key in properties:
if not len(key):
raise RestException('Property names must be at least one'
' character long.')
if '.' in key or key[0] == '$':
raise RestException(
'The property name %s must not contain'
' a period or begin with a dollar sign.' % key)
data.append({
'name': name,
'description': description,
'metadata': properties,
'geometry': feature['geometry']
})
user = self.getCurrentUser()
items = []
for datum in data:
newItem = Item().createItem(folder=folder,
name=datum['name'],
creator=user,
description=datum['description'])
Item().setMetadata(newItem, datum['metadata'])
newItem[GEOSPATIAL_FIELD] = {'geometry': datum['geometry']}
newItem = Item().updateItem(newItem)
items.append(newItem)
return items
@access.public
@filtermodel(Item)
@autoDescribeRoute(
Description('Search for an item by geospatial data.').jsonParam(
'q', 'Search query as a JSON object.').pagingParams(
defaultSort='lowerName').errorResponse())
def find(self, q, limit, offset, sort):
return self._find(q, limit, offset, sort)
@access.public
@filtermodel(Item)
@autoDescribeRoute(
Description('Search for items that intersects with a GeoJSON object.').
param('field',
'Name of field containing GeoJSON on which to search.',
strip=True).jsonParam(
'geometry',
'Search query condition as a GeoJSON object.').pagingParams(
defaultSort='lowerName').errorResponse())
def intersects(self, field, geometry, limit, offset, sort):
try:
GeoJSON.to_instance(geometry, strict=True)
except (TypeError, ValueError):
raise RestException(
"Invalid GeoJSON passed as 'geometry' parameter.")
if field[:3] != '%s.' % GEOSPATIAL_FIELD:
field = '%s.%s' % (GEOSPATIAL_FIELD, field)
query = {field: {'$geoIntersects': {'$geometry': geometry}}}
return self._find(query, limit, offset, sort)
def _getGeometry(self, geometry):
try:
GeoJSON.to_instance(geometry, strict=True)
if geometry['type'] != 'Point':
raise ValueError
return geometry
except (TypeError, ValueError):
raise RestException(
"Invalid GeoJSON passed as 'geometry' parameter.")
@access.public
@filtermodel(Item)
@autoDescribeRoute(
Description(
'Search for items that are in proximity to a GeoJSON point.').
param('field',
'Name of field containing GeoJSON on which to search.',
strip=True).jsonParam(
'geometry',
'Search query condition as a GeoJSON point.').param(
'maxDistance',
'Limits results to items that are at most this distance '
'in meters from the GeoJSON point.',
required=False,
dataType='number').
param('minDistance',
'Limits results to items that are at least this distance '
'in meters from the GeoJSON point.',
required=False,
dataType='number').param(
'ensureIndex',
'Create a 2dsphere index on the field on which to search '
'if one does not exist.',
required=False,
dataType='boolean',
default=False).pagingParams(
defaultSort='lowerName').errorResponse().errorResponse(
'Field on which to search was not indexed.').
errorResponse('Index creation was denied.', 403).notes(
"Field on which to search be indexed by a 2dsphere index."
" Anonymous users may not use 'ensureIndex' to create such an index."
))
def near(self, field, geometry, maxDistance, minDistance, ensureIndex,
limit, offset, sort):
condition = {'$geometry': self._getGeometry(geometry)}
if maxDistance is not None:
if maxDistance < 0:
raise RestException('maxDistance must be positive.')
condition['$maxDistance'] = maxDistance
if minDistance is not None:
if minDistance < 0:
raise RestException('minDistance must be positive.')
condition['$minDistance'] = minDistance
if field[:3] != '%s.' % GEOSPATIAL_FIELD:
field = '%s.%s' % (GEOSPATIAL_FIELD, field)
if ensureIndex:
user = self.getCurrentUser()
if not user:
raise RestException('Index creation denied.', 403)
Item().collection.create_index([(field, GEOSPHERE)])
query = {field: {'$near': condition}}
try:
return self._find(query, limit, offset, sort)
except OperationFailure:
raise RestException(
"Field '%s' must be indexed by a 2dsphere index." % field)
_RADIUS_OF_EARTH = 6378137.0 # average in meters
@access.public
@filtermodel(Item)
@autoDescribeRoute(
Description(
'Search for items that are entirely within either a GeoJSON'
' polygon or a circular region.').param(
'field',
'Name of field containing GeoJSON on which to search.',
strip=True).jsonParam(
'geometry',
'Search query condition as a GeoJSON polygon.',
required=False).jsonParam(
'center',
'Center of search radius as a GeoJSON point.',
required=False,
requireObject=True).param('radius',
'Search radius in meters.',
required=False,
dataType='number').
pagingParams(defaultSort='lowerName').errorResponse().errorResponse(
'Field on which to search was not indexed.').errorResponse(
'Index creation was denied.', 403).notes(
"Either parameter 'geometry' or both parameters 'center' "
" and 'radius' are required."))
def within(self, field, geometry, center, radius, limit, offset, sort):
if geometry is not None:
try:
GeoJSON.to_instance(geometry, strict=True)
if geometry['type'] != 'Polygon':
raise ValueError
except (TypeError, ValueError):
raise RestException(
"Invalid GeoJSON passed as 'geometry' parameter.")
condition = {'$geometry': geometry}
elif center is not None and radius is not None:
try:
radius /= self._RADIUS_OF_EARTH
if radius < 0.0:
raise ValueError
except ValueError:
raise RestException("Parameter 'radius' must be a number.")
try:
GeoJSON.to_instance(center, strict=True)
if center['type'] != 'Point':
raise ValueError
except (TypeError, ValueError):
raise RestException(
"Invalid GeoJSON passed as 'center' parameter.")
condition = {'$centerSphere': [center['coordinates'], radius]}
else:
raise RestException(
"Either parameter 'geometry' or both parameters"
" 'center' and 'radius' are required.")
if field[:3] != '%s.' % GEOSPATIAL_FIELD:
field = '%s.%s' % (GEOSPATIAL_FIELD, field)
query = {field: {'$geoWithin': condition}}
return self._find(query, limit, offset, sort)
@access.public
@filtermodel(Item)
@autoDescribeRoute(
Description('Get an item and its geospatial data by ID.').modelParam(
'id', 'The ID of the item.', model='item',
level=AccessType.READ).errorResponse(
'ID was invalid.').errorResponse(
'Read access was denied for the item.', 403).deprecated())
def getGeospatial(self, item):
# Deprecated -- we use the modern filtering mechanisms now to include the geo field
return item
@access.user
@filtermodel(Item)
@autoDescribeRoute(
Description('Set geospatial fields on an item.').notes(
'Set geospatial fields to null to delete them.').modelParam(
'id',
'The ID of the item.',
model='item',
level=AccessType.WRITE).jsonParam(
'geospatial',
'A JSON object containing the geospatial fields to add.',
paramType='body').errorResponse('ID was invalid.').
errorResponse('Invalid JSON was passed in request body.').
errorResponse('Geospatial key name was invalid.').errorResponse(
'Geospatial field did not contain valid GeoJSON.').errorResponse(
'Write access was denied for the item.', 403))
def setGeospatial(self, item, geospatial):
for k, v in six.viewitems(geospatial):
if '.' in k or k[0] == '$':
raise RestException('Geospatial key name %s must not contain a'
' period or begin with a dollar sign.' % k)
if v:
try:
GeoJSON.to_instance(v, strict=True)
except ValueError:
raise RestException('Geospatial field with key %s does not'
' contain valid GeoJSON: %s' % (k, v))
if GEOSPATIAL_FIELD not in item:
item[GEOSPATIAL_FIELD] = {}
item[GEOSPATIAL_FIELD].update(six.viewitems(geospatial))
keys = [
k for k, v in six.viewitems(item[GEOSPATIAL_FIELD]) if v is None
]
for key in keys:
del item[GEOSPATIAL_FIELD][key]
return Item().updateItem(item)
def _find(self, query, limit, offset, sort):
"""
Helper to search the geospatial data of items and return the filtered
fields and geospatial data of the matching items.
:param query: geospatial search query.
:type query: dict[str, unknown]
:param limit: maximum number of matching items to return.
:type limit: int
:param offset: offset of matching items to return.
:type offset: int
:param sort: field by which to sort the matching items
:type sort: str
:returns: filtered fields of the matching items with geospatial data
appended to the 'geo' field of each item.
:rtype : list[dict[str, unknown]]
"""
cursor = Item().find(query, sort=sort)
return list(Item().filterResultsByPermission(cursor,
self.getCurrentUser(),
AccessType.READ, limit,
offset))
class ActivitiesResource(Resource):
def __init__(self):
super(ActivitiesResource, self).__init__()
self.resourceName = 'activities'
self.route('GET', (':folderId',), self.getActivitiesOfFolder)
self.route('POST', (':folderId',), self.addActivitiesToFolder)
self.route('PUT', (':activityId',), self.updateActivity)
self.route('DELETE', (':activityId',), self.deleteActivity)
self.route('GET', ('export', ':folderId',), self.exportKPF)
@autoDescribeRoute(
Description('')
.modelParam('folderId', model=Folder, level=AccessType.READ)
.errorResponse()
.errorResponse('Read access was denied on the item.', 403)
)
@access.user
def getActivitiesOfFolder(self, folder, params):
cursor = Activities().findByFolder(folder)
return list(cursor)
@autoDescribeRoute(
Description('')
.modelParam('folderId', model=Folder, level=AccessType.WRITE)
.jsonParam('data', 'The activity content', requireObject=True, paramType='body')
.errorResponse()
.errorResponse('Read access was denied on the item.', 403)
)
@access.user
def addActivitiesToFolder(self, folder, data, params):
data['folderId'] = folder['_id']
return Activities().save(data)
@autoDescribeRoute(
Description('')
.modelParam('activityId', model=Activities)
.jsonParam('data', 'The activity content', requireObject=True, paramType='body')
.errorResponse()
.errorResponse('Read access was denied on the item.', 403)
)
@access.user
def updateActivity(self, activities, data, params):
data.pop('_id', None)
data.pop('folderId', None)
activities.update(data)
return Activities().save(activities)
@autoDescribeRoute(
Description('')
.modelParam('activityId', model=Activities)
.errorResponse()
.errorResponse('Read access was denied on the item.', 403)
)
@access.user
def deleteActivity(self, activities, params):
Activities().remove(activities)
return ''
@autoDescribeRoute(
Description('')
.modelParam('folderId', model=Folder, level=AccessType.READ)
.errorResponse()
.errorResponse('Read access was denied on the item.', 403)
)
@access.user
@access.cookie
@rawResponse
def exportKPF(self, folder, params):
setResponseHeader('Content-Type', 'text/plain')
setResponseHeader('Content-Disposition', 'attachment; filename=activities.kpf')
return self.generateKPFContent(folder)
@staticmethod
def generateKPFContent(folder):
cursor = Activities().findByFolder(folder)
output = []
for activity in cursor:
del activity['_id']
del activity['folderId']
activity = yaml.safe_dump(activity, default_flow_style=True,
width=1000).rstrip()
output.append('- {{ act: {0} }}'.format(activity))
def gen():
yield '\n'.join(output)
return gen
class Workspace(Resource):
def __init__(self):
super(Workspace, self).__init__()
self.resourceName = 'workspace'
self.route('GET', (), self.getWorkspace)
self.route('POST', (), self.createWorkspace)
self.route('DELETE', (':id', ), self.deleteWorkspace)
@access.user
@autoDescribeRoute(Description('').errorResponse())
def getWorkspace(self, params):
user = self.getCurrentUser()
workspaces = list(Item().findWithPermissions(
{
'meta.resonanteco_workspace': True,
},
user=user,
level=AccessType.READ))
workspaces.sort(key=lambda item: item['updated'], reverse=True)
return workspaces
@access.user
@autoDescribeRoute(
Description('').jsonParam('workspace',
'',
requireObject=True,
required=True,
paramType='body').errorResponse())
def createWorkspace(self, workspace, params):
user = self.getCurrentUser()
private = Folder().createFolder(user,
'Private',
parentType='user',
public=False,
creator=user,
reuseExisting=True)
resonantEcoFolder = Folder().createFolder(private,
'ResonantEco',
creator=user,
reuseExisting=True)
workspacesFolder = Folder().createFolder(resonantEcoFolder,
'Workspaces',
creator=user,
reuseExisting=True)
# id = ObjectId.from_datetime(datetime.datetime.now())
workspaceItem = Item().createItem(workspace['name'], user,
workspacesFolder)
return Item().setMetadata(
workspaceItem, {
'resonanteco_workspace': True,
'datasets': workspace['datasets'],
'filter': workspace['filter']
})
@access.user
@autoDescribeRoute(
Description('').modelParam('id',
model=Item,
destName='workspace',
level=AccessType.WRITE).errorResponse())
def deleteWorkspace(self, workspace, params):
Item().remove(workspace)
class ImageResource(Resource):
def __init__(self):
super().__init__()
self.resourceName = 'image'
self.cp_config = {
'tools.staticdir.on': True,
'tools.staticdir.index': 'index.html'
}
# supported image files
self.supported_img_exts = [".jpg", ".jpeg", ".png", ".svs", ".tiff"]
self.route('GET', (), handler=self.getImageList)
self.route('GET', (':image_id', ), self.getImage)
self.route('GET', ('thumbnail', ), self.getThumbnail)
self.route('GET', (
'dzi',
':image_id',
), self.dzi)
self.route('GET', ('dzi', ':image_id', ':level', ':tfile'), self.tile)
self.slide_cache = SlidesCache(None, None)
@staticmethod
def __load_slides(file):
printOk2(file)
assetstore = Assetstore().load(file['assetstoreId'])
slides, associated_images, slide_properties, slide_mpp = \
load_slide(slidefile=os.path.join(assetstore['root'], file['path']),
tile_size=126)
return slides
def find_label_id(self, folder, name):
collection_model = Collection()
labels = Folder().fileList(doc=folder,
user=self.getCurrentUser(),
data=False,
includeMetadata=True,
mimeFilter=['application/json'])
for labelname, label in labels:
labelname = os.path.splitext(labelname)[0]
printOk2("labelname: " + labelname + " " + name)
if labelname == name:
return label['_id']
@staticmethod
def __set_mime_type(ext):
if ext == ".jpg" or ext == ".jpeg":
return "image/jpeg"
elif ext == ".png" or ext == ".PNG":
return "image/png"
elif ext in ['.svs', '.tiff']:
return "application/octet-stream"
def __filter(self, items, exts):
ret = []
for item in items:
name, ext = os.path.splitext(item['name'])
ext = ext.lower()
if ext in exts:
item['mimeType'] = self.__set_mime_type(ext)
ret.append(item)
return ret
@access.public
@autoDescribeRoute(
Description('Get image list').param('folderId', 'folder id').param(
'limit', 'Number of assignments to return').param(
'offset',
'offset from 0th assignment to start looking for assignments'))
@rest.rawResponse
@trace
def getImageList(self, folderId, limit, offset):
printOk('getImageList() was called!')
limit, offset = int(limit), int(offset)
self.user = self.getCurrentUser()
folder = Folder().load(folderId,
level=AccessType.READ,
user=self.getCurrentUser())
items = Folder().childItems(folder, limit=limit, offset=offset)
items = self.__filter(items, exts=self.supported_img_exts)
ret_files = []
for item in items:
# TODO: remove this function
filename = os.path.splitext(item['name'])[0]
printOk("filename: " + filename)
ret_files.append(item)
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(ret_files)
@access.public
@autoDescribeRoute(
Description('Get dzi').param('image_id', 'image file id'))
@rest.rawResponse
@trace
def dzi(self, image_id):
printOk('getDzi() was called!')
item = Item().load(image_id, level=AccessType.READ, user=self.user)
file = self.__get_file(item, item['name'])
slides = self.__load_slides(file)
resp = slides['slide'].get_dzi('jpeg')
cherrypy.response.headers["Content-Type"] = "application/xml"
self.slide_cache = SlidesCache(image_id, slides)
return resp
def __get_file(self, item, fname):
files = Item().fileList(item, user=self.getCurrentUser(), data=False)
for filepath, file in files:
if fname in file['name']:
return file
@access.public
@autoDescribeRoute(
Description('Get image').param('image_id', 'image file id'))
@rest.rawResponse
@trace
def getImage(self, image_id):
item = Item().load(image_id, level=AccessType.READ, user=self.user)
file = self.__get_file(item, item['name'])
cherrypy.response.headers["Content-Type"] = "application/png"
return File().download(file, headers=False)
@access.public
@autoDescribeRoute(Description('get tiles'))
@rest.rawResponse
@trace
def tile(self, image_id, level, tfile):
image_id = re.search(r'(.*)_files', image_id).group(1)
# retreive from cache if already found.
if image_id != self.slide_cache.image_id:
item = Item().load(image_id, level=AccessType.READ, user=self.user)
file = self.__get_file(item, item['name'])
slides = self.__load_slides(file)
self.slide_cache = SlidesCache(image_id, slides)
else:
slides = self.slide_cache.slides
pos, _format = tfile.split('.')
col, row = pos.split('_')
_format = _format.lower()
if _format != 'jpeg' and _format != 'png':
# Not supported by Deep Zoom
cherrypy.response.status = 404
tile_image = slides['slide'].get_tile(int(level), (int(col), int(row)))
buf = PILBytesIO()
tile_image.save(buf, _format, quality=90)
cherrypy.response.headers["Content-Type"] = 'image/%s' % _format
resp = buf.getvalue()
return resp
@access.public
@autoDescribeRoute(
Description('Get thumbnail by size').param(
'image_id',
'image file id').param('w',
'thumbnail width').param('h',
'thumbnail height',
required=False))
@rest.rawResponse
@trace
def getThumbnail(self, image_id, w, h=None):
item = Item().load(image_id, level=AccessType.READ, user=self.user)
start_time = timeit.default_timer()
if not h:
filename = "thumbnail_{}".format(w)
else:
filename = "thumbnail_{}x{}".format(w, h)
file = self.__get_file(item, filename)
print(file)
if not file:
file = self.__create_thumbnail(item, w, h)
elapsed_time = timeit.default_timer() - start_time
printOk2("thumbnail file just created in {}".format(elapsed_time))
cherrypy.response.headers["Content-Type"] = "application/jpeg"
return File().download(file, headers=False)
def __create_thumbnail(self, item, w, h):
w = int(w)
file = self.__get_file(item, item['name'])
with File().open(file) as f:
image = Image.open(BytesIO(f.read()))
# incase we are currently processing png images, which have RGBA.
# we convert to RGB, cos we save the thumbnail into a .jpg which cannot handle A channel
image = image.convert("RGB")
if not h:
width, height = image.size
h = (height / width) * w
h = int(h)
image.thumbnail((w, h))
buf = PILBytesIO()
image.save(buf, "jpeg", quality=100)
thumbnailFile = File().createFile(
size=0,
item=item,
name="thumbnail_{}x{}.jpg".format(w, h),
creator=self.user,
assetstore=Assetstore().getCurrent(),
mimeType="application/jpeg")
writeBytes(self.user, thumbnailFile, buf.getvalue())
thumbnailFile = self.__get_file(item,
"thumbnail_{}x{}.jpg".format(w, h))
return thumbnailFile
