def _loadMetadata(self):
fields = [key.split('_', 1)[1].lower() for key in
dir(libtiff_ctypes.tiff_h) if key.startswith('TIFFTAG_')]
info = {}
for field in fields:
try:
value = self._tiffFile.GetField(field)
if value is not None:
info[field] = value
except TypeError as err:
logger.debug('Loading field "%s" in directory number %d '
'resulted in TypeError - "%s"',
field, self._directoryNum, err)
for func in self.CoreFunctions[2:]:
if hasattr(self._tiffFile, func):
value = getattr(self._tiffFile, func)()
if value:
info[func.lower()] = value
self._tiffInfo = info
self._tileWidth = info.get('tilewidth')
self._tileHeight = info.get('tilelength')
self._imageWidth = info.get('imagewidth')
self._imageHeight = info.get('imagelength')
self.parse_image_description(info.get('imagedescription', ''))
def __init__(self, filePath, directoryNum):
"""
Create a new reader for a tiled image file directory in a TIFF file.
:param filePath: A path to a TIFF file on disk.
:type filePath: str
:param directoryNum: The number of the TIFF image file directory to
open.
:type directoryNum: int
:raises: InvalidOperationTiffException or IOTiffException or
ValidationTiffException
"""
# TODO how many to keep in the cache
# create local cache to store Jpeg tables and
# getTileByteCountsType
self.cache = LRUCache(10)
self._tiffFile = None
self._open(filePath, directoryNum)
self._loadMetadata()
logger.debug('TiffDirectory %d Information %r',
directoryNum, self._tiffInfo)
try:
self._validate()
except ValidationTiffException:
self._close()
raise
def importFile(self, item, path, user, name=None, mimeType=None, **kwargs):
"""
Import a single file from the filesystem into the assetstore.
:param item: The parent item for the file.
:type item: dict
:param path: The path on the local filesystem.
:type path: str
:param user: The user to list as the creator of the file.
:type user: dict
:param name: Name for the file. Defaults to the basename of ``path``.
:type name: str
:param mimeType: MIME type of the file if known.
:type mimeType: str
:returns: The file document that was created.
"""
logger.debug('Importing file %s to item %s on filesystem assetstore %s',
path, item['_id'], self.assetstore['_id'])
stat = os.stat(path)
name = name or os.path.basename(path)
file = File().createFile(
name=name, creator=user, item=item, reuseExisting=True, assetstore=self.assetstore,
mimeType=mimeType, size=stat.st_size, saveFile=False)
file['path'] = os.path.abspath(os.path.expanduser(path))
file['mtime'] = stat.st_mtime
file['imported'] = True
file = File().save(file)
logger.debug('Imported file %s to item %s on filesystem assetstore %s',
path, item['_id'], self.assetstore['_id'])
return file
def adjustDBUri(cls, uri, *args, **kwargs):
"""
Adjust a uri to match the form sqlite requires. This can convert a
Girder resource path to an aprpopriate physical file reference.
:param uri: the uri to adjust.
:returns: the adjusted uri
"""
uri = super(SqliteSAConnector, cls).adjustDBUri(uri, *args, **kwargs)
if '://' in uri:
uri = uri.split('://', 1)[0] + ':////' + uri.split('://', 1)[1].lstrip('/')
uri = super(SqliteSAConnector, cls).adjustDBUri(uri, *args, **kwargs)
# If we have a Girder resource path, convert it. If this looks like a
# file but doesn't exist, check if it is a resource path. If this is
# not a resoruce path to a file that we can read directly, treat this
# the same as a missing file.
if (':///' in uri and not os.path.exists(uri.split(':///', 1)[1])):
resourcepath = path_util.lookUpPath(
uri.split(':///', 1)[1], test=True, filter=False, force=True)
if resourcepath and resourcepath['model'] == 'file':
file = resourcepath['document']
adapter = File().getAssetstoreAdapter(file)
if hasattr(adapter, 'fullPath'):
filepath = adapter.fullPath(file)
if os.path.exists(filepath):
uri = uri.split(':///', 1)[0] + ':///' + filepath
log.debug('Using Girder file for SQLite database')
return uri
def refine(self, params):
sid = str(params['item']['_id'])
pos_uuids = params['pos_uuids']
neg_uuids = params['neg_uuids'] if params['neg_uuids'] is not None else []
if len(pos_uuids) == 0:
raise RestException('No positive UUIDs given.')
with self.controller:
if not self.controller.has_session_uuid(sid):
raise RestException('Session ID %s not found.' % sid, 404)
iqrs = self.controller.get_session(sid)
iqrs.lock.acquire() # lock BEFORE releasing controller
try:
descriptor_index = self._descriptorIndexFromSessionId(sid)
neighbor_index = self._nearestNeighborIndex(sid, descriptor_index)
if descriptor_index is None or neighbor_index is None:
logger.error('Unable to compute descriptor or neighbor index from sid %s.' % sid)
raise RestException('Unable to compute descriptor or neighbor index from sid %s.' % sid, 500)
# Get appropriate descriptor elements from index for
# setting new adjudication state.
try:
pos_descrs = set(descriptor_index.get_many_descriptors(pos_uuids))
neg_descrs = set(descriptor_index.get_many_descriptors(neg_uuids))
except KeyError as ex:
logger.warn(traceback.format_exc())
raise RestException('Descriptor UUID %s not found in index.' % ex, 404)
# if a new classifier should be made upon the next
# classification request.
diff_pos = pos_descrs.symmetric_difference(iqrs.positive_descriptors)
diff_neg = neg_descrs.symmetric_difference(iqrs.negative_descriptors)
if diff_pos or diff_neg:
logger.debug("[%s] session Classifier dirty", sid)
self.session_classifier_dirty[sid] = True
logger.info("[%s] Setting adjudications", sid)
iqrs.positive_descriptors = pos_descrs
iqrs.negative_descriptors = neg_descrs
logger.info("[%s] Updating working index", sid)
iqrs.update_working_index(neighbor_index)
logger.info("[%s] Refining", sid)
iqrs.refine()
finally:
iqrs.lock.release()
return sid
def __call__(self, op, path, *args, **kwargs):
"""
Generically allow logging and error handling for any operation.
:param op: operation to perform.
:param path: path within the fuse (e.g., '', '/user', '/user/<name>',
etc.).
"""
logger.debug('-> %s %s %s', op, path, repr(args))
ret = '[exception]'
try:
ret = getattr(self, op)(path, *args, **kwargs)
return ret
except Exception as e:
# Log all exceptions and then reraise them
if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
logger.debug('-- %s %r', op, e)
else:
logger.exception('-- %s', op)
raise e
finally:
if op != 'read':
logger.debug('<- %s %s', op, repr(ret))
else:
logger.debug('<- %s (length %d) %r', op, len(ret), ret[:16])
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
logger.debug('Mounting %s\n' % mountpoint)
super(FUSELogError, self).__init__(operations, mountpoint, *args, **kwargs)
logger.debug('Mounted %s\n' % mountpoint)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' % (
mountpoint, ))
Setting().unset(SettingKey.GIRDER_MOUNT_INFORMATION)
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
logger.debug('Mounting %s\n' % mountpoint)
super(FUSELogError, self).__init__(operations, mountpoint, *args,
**kwargs)
logger.debug('Mounted %s\n' % mountpoint)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' %
(mountpoint, ))
Setting().unset(SettingKey.GIRDER_MOUNT_INFORMATION)
def _get_attrs_via_head(obj, url):
name = obj['filename']
size = obj['filesize']
try:
req = Request(url)
req.get_method = lambda: 'HEAD'
resp = urlopen(req)
except HTTPError as err:
logger.debug(str(err))
return name, size
content_disposition = resp.getheader('Content-Disposition')
if content_disposition:
fname = _CNTDISP_REGEX.search(content_disposition)
if fname:
name = fname.groups()[0]
content_length = resp.getheader('Content-Length')
if content_length:
size = int(content_length)
return name, size
def importFile(self, item, path, user, name=None, mimeType=None, **kwargs):
"""
Import a single file from the filesystem into the assetstore.
:param item: The parent item for the file.
:type item: dict
:param path: The path on the local filesystem.
:type path: str
:param user: The user to list as the creator of the file.
:type user: dict
:param name: Name for the file. Defaults to the basename of ``path``.
:type name: str
:param mimeType: MIME type of the file if known.
:type mimeType: str
:returns: The file document that was created.
"""
logger.debug(
'Importing file %s to item %s on filesystem assetstore %s', path,
item['_id'], self.assetstore['_id'])
stat = os.stat(path)
name = name or os.path.basename(path)
file = File().createFile(name=name,
creator=user,
item=item,
reuseExisting=True,
assetstore=self.assetstore,
mimeType=mimeType,
size=stat.st_size,
saveFile=False)
file['path'] = os.path.abspath(os.path.expanduser(path))
file['mtime'] = stat.st_mtime
file['imported'] = True
file = File().save(file)
logger.debug('Imported file %s to item %s on filesystem assetstore %s',
path, item['_id'], self.assetstore['_id'])
return file
def get_package_list(path, package=None, isChild=False):
""""""
if package is None:
package = {}
package_pid = get_package_pid(path)
logger.debug('Found package PID of {}.'.format(package_pid))
docs = get_documents(package_pid)
# Filter the Solr result by TYPE so we can construct the package
metadata = extract_metadata_docs(docs)
data = extract_data_docs(docs)
children = extract_resource_docs(docs)
# Determine the folder name. This is usually the title of the metadata file
# in the package but when there are multiple metadata files in the package,
# we need to figure out which one is the 'main' or 'documenting' one.
primary_metadata = [doc for doc in metadata if 'documents' in doc]
check_multiple_metadata(primary_metadata)
data += [doc for doc in metadata if doc['identifier'] != primary_metadata[0]['identifier']]
fileList = get_package_files(data, metadata, primary_metadata)
# Add a new entry in the package structure
# if isChild:
# package[-1][primary_metadata[0]['title']] = {'fileList': []}
# else:
package[primary_metadata[0]['title']] = {'fileList': []}
package[primary_metadata[0]['title']]['fileList'].append(fileList)
if children is not None and len(children) > 0:
for child in children:
get_package_list(child['identifier'], package[primary_metadata[0]['title']], True)
return package
def import_recursive(job):
try:
root = job['kwargs']['root']
token = job['kwargs']['token']
jobModel = ModelImporter.model('job', 'jobs')
userModel = ModelImporter.model('user')
user = userModel.load(job['userId'], force=True)
childModel = ModelImporter.model('cohort', 'digital_slide_archive')
children = list(
ModelImporter.model('folder').childFolders(root,
'collection',
user=user))
count = len(children)
progress = 0
job = jobModel.updateJob(job,
log='Started TCGA import\n',
status=JobStatus.RUNNING,
progressCurrent=progress,
progressTotal=count)
logger.info('Starting recursive TCGA import')
for child in children:
progress += 1
try:
msg = 'Importing "%s"' % child.get('name', '')
job = jobModel.updateJob(job,
log=msg,
progressMessage=msg + '\n',
progressCurrent=progress)
logger.debug(msg)
childModel.importDocument(child,
recurse=True,
user=user,
token=token,
job=job)
job = jobModel.load(id=job['_id'], force=True)
# handle any request to stop execution
if (not job or job['status']
in (JobStatus.CANCELED, JobStatus.ERROR)):
logger.info('TCGA import job halted with')
return
except ValidationException:
logger.warning('Failed to import %s' % child.get('name', ''))
logger.info('Starting recursive TCGA import')
job = jobModel.updateJob(job,
log='Finished TCGA import\n',
status=JobStatus.SUCCESS,
progressCurrent=count,
progressMessage='Finished TCGA import')
except Exception as e:
logger.exception('Importing TCGA failed with %s' % str(e))
job = jobModel.updateJob(job,
log='Import failed with %s\n' % str(e),
status=JobStatus.ERROR)
def createEmptyResource(self, name):
logger.debug('%s -> createEmptyResource(%s)' % (self.getRefUrl(), name))
return FolderResource.createEmptyResource(self, name)
def __init__(self, path, **kwargs):
"""
Initialize the tile class. See the base class for other available
parameters.
:param path: a filesystem path for the tile source.
"""
super(TiffFileTileSource, self).__init__(path, **kwargs)
largeImagePath = self._getLargeImagePath()
lastException = None
# Associated images are smallish TIFF images that have an image
# description and are not tiled. They have their own TIFF directory.
# Individual TIFF images can also have images embedded into their
# directory as tags (this is a vendor-specific method of adding more
# images into a file) -- those are stored in the individual
# directories' _embeddedImages field.
self._associatedImages = {}
# Query all know directories in the tif file. Only keep track of
# directories that contain tiled images.
alldir = []
for directoryNum in itertools.count(): # pragma: no branch
try:
td = TiledTiffDirectory(largeImagePath, directoryNum)
except ValidationTiffException as exc:
lastException = exc
self._addAssociatedImage(largeImagePath, directoryNum)
continue
except TiffException as exc:
if not lastException:
lastException = exc
break
if not td.tileWidth or not td.tileHeight:
continue
# Calculate the tile level, where 0 is a single tile, 1 is up to a
# set of 2x2 tiles, 2 is 4x4, etc.
level = int(
math.ceil(
math.log(
max(
float(td.imageWidth) / td.tileWidth,
float(td.imageHeight) / td.tileHeight)) /
math.log(2)))
if level < 0:
continue
# Store information for sorting with the directory.
alldir.append((td.tileWidth * td.tileHeight, level,
td.imageWidth * td.imageHeight, directoryNum, td))
# If there are no tiled images, raise an exception.
if not len(alldir):
msg = 'File %s didn\'t meet requirements for tile source: %s' % (
largeImagePath, lastException)
logger.debug(msg)
raise TileSourceException(msg)
# Sort the known directories by image area (width * height). Given
# equal area, sort by the level.
alldir.sort()
# The highest resolution image is our preferred image
highest = alldir[-1][-1]
directories = {}
# Discard any images that use a different tiling scheme than our
# preferred image
for tdir in alldir:
td = tdir[-1]
level = tdir[1]
if (td.tileWidth != highest.tileWidth
or td.tileHeight != highest.tileHeight):
continue
# If a layer's image is not a multiple of the tile size, it should
# be near a power of two of the highest resolution image.
if (((td.imageWidth % td.tileWidth)
and not nearPowerOfTwo(td.imageWidth, highest.imageWidth)) or
((td.imageHeight % td.tileHeight)
and not nearPowerOfTwo(td.imageHeight, highest.imageHeight))):
continue
directories[level] = td
if not len(directories) or (len(directories) < 2
and max(directories.keys()) + 1 > 4):
raise TileSourceException(
'Tiff image must have at least two levels.')
# Sort the directories so that the highest resolution is the last one;
# if a level is missing, put a None value in its place.
self._tiffDirectories = [
directories.get(key) for key in range(max(directories.keys()) + 1)
]
self.tileWidth = highest.tileWidth
self.tileHeight = highest.tileHeight
self.levels = len(self._tiffDirectories)
self.sizeX = highest.imageWidth
self.sizeY = highest.imageHeight
def refine(self, params):
sid = str(params['item']['_id'])
pos_uuids = params['pos_uuids']
neg_uuids = params['neg_uuids'] if params[
'neg_uuids'] is not None else []
if len(pos_uuids) == 0:
raise RestException('No positive UUIDs given.')
with self.controller:
if not self.controller.has_session_uuid(sid):
raise RestException('Session ID %s not found.' % sid, 404)
iqrs = self.controller.get_session(sid)
iqrs.lock.acquire() # lock BEFORE releasing controller
try:
descriptor_set = self._descriptorSetFromSessionId(sid)
neighbor_index = self._nearestNeighborIndex(sid, descriptor_set)
if descriptor_set is None or neighbor_index is None:
logger.error(
'Unable to compute descriptor or neighbor index from sid %s.'
% sid)
raise RestException(
'Unable to compute descriptor or neighbor index from sid %s.'
% sid, 500)
# Get appropriate descriptor elements from index for
# setting new adjudication state.
try:
pos_descrs = set(
descriptor_set.get_many_descriptors(pos_uuids))
neg_descrs = set(
descriptor_set.get_many_descriptors(neg_uuids))
except KeyError as ex:
logger.warn(traceback.format_exc())
raise RestException(
'Descriptor UUID %s not found in '
'descriptor set.' % ex, 404)
# if a new classifier should be made upon the next
# classification request.
diff_pos = pos_descrs.symmetric_difference(
iqrs.positive_descriptors)
diff_neg = neg_descrs.symmetric_difference(
iqrs.negative_descriptors)
if diff_pos or diff_neg:
logger.debug("[%s] session Classifier dirty", sid)
self.session_classifier_dirty[sid] = True
logger.info("[%s] Setting adjudications", sid)
iqrs.positive_descriptors = pos_descrs
iqrs.negative_descriptors = neg_descrs
logger.info("[%s] Updating working index", sid)
iqrs.update_working_set(neighbor_index)
logger.info("[%s] Refining", sid)
iqrs.refine()
finally:
iqrs.lock.release()
return sid
def genRESTEndPointsForSlicerCLIsForItem(restResource,
cliItem,
registerNamedRoute=False):
"""Generates REST end points for slicer CLIs placed in subdirectories of a
given root directory and attaches them to a REST resource with the given
name.
For each CLI, it creates:
* a GET Route (<apiURL>/`restResourceName`/<cliRelativePath>/xmlspec)
that returns the xml spec of the CLI
* a POST Route (<apiURL>/`restResourceName`/<cliRelativePath>/run)
that runs the CLI
It also creates a GET route (<apiURL>/`restResourceName`) that returns a
list of relative routes to all CLIs attached to the generated REST resource
Parameters
----------
restResource : a dockerResource
REST resource to which the end-points should be attached
cliItem : CliItem
"""
# validate restResource argument
if not isinstance(restResource, Resource):
raise Exception('restResource must be a Docker Resource')
try:
handler = genHandlerToRunDockerCLI(cliItem)
# define CLI handler function
cliRunHandler = boundHandler(restResource)(handler)
cliRunHandlerName = 'run_%s' % cliItem._id
setattr(restResource, cliRunHandlerName, cliRunHandler)
restRunPath = ('cli', str(cliItem._id), 'run')
restResource.route('POST', restRunPath, cliRunHandler)
if registerNamedRoute:
restNamedRunPath = (cliItem.restBasePath, cliItem.name, 'run')
restResource.route('POST', restNamedRunPath, cliRunHandler)
def undoFunction():
try:
restResource.removeRoute('POST', restRunPath, cliRunHandler)
if registerNamedRoute:
restResource.removeRoute('POST', restNamedRunPath,
cliRunHandler)
delattr(restResource, cliRunHandlerName)
except Exception:
logger.exception('Failed to remove route')
# store new rest endpoint
restResource.storeEndpoints(cliItem.image, cliItem.name, undoFunction)
logger.debug('Created REST endpoints for %s', cliItem.name)
except Exception:
logger.exception('Failed to create REST endpoints for %r',
cliItem.name)
return restResource
def save(self, annotation, *args, **kwargs):
"""
When saving an annotation, override the collection insert_one and
replace_one methods so that we don't save the elements with the main
annotation. Still use the super class's save method, so that all of
the triggers are fired as expected and cancelling and modifications can
be done as needed.
Because Mongo doesn't support transactions, a version number is stored
with the annotation and with the associated elements. This is used to
add the new elements first, then update the annotation, and delete the
old elements. The allows version integrity if another thread queries
the annotation at the same time.
:param annotation: the annotation document to save.
:returns: the saved document. If it is a new document, the _id has
been added.
"""
starttime = time.time()
replace_one = self.collection.replace_one
insert_one = self.collection.insert_one
version = self.model('annotationelement',
'large_image').getNextVersionValue()
if '_id' not in annotation:
oldversion = None
else:
# We read the old version from the existing record, because we
# don't want to trust that the input _version has not been altered
# or is present.
oldversion = self.collection.find_one({
'_id': annotation['_id']
}).get('_version')
annotation['_version'] = version
_elementQuery = annotation.pop('_elementQuery', None)
def replaceElements(query, doc, *args, **kwargs):
self.model('annotationelement', 'large_image').updateElements(doc)
elements = doc['annotation'].pop('elements', None)
ret = replace_one(query, doc, *args, **kwargs)
if elements:
doc['annotation']['elements'] = elements
self.model('annotationelement',
'large_image').removeOldElements(doc, oldversion)
return ret
def insertElements(doc, *args, **kwargs):
elements = doc['annotation'].pop('elements', None)
# When creating an annotation, there is a window of time where the
# elements aren't set (this is unavoidable without database
# transactions, as we need the annotation's id to set the
# elements).
ret = insert_one(doc, *args, **kwargs)
if elements is not None:
doc['annotation']['elements'] = elements
self.model('annotationelement',
'large_image').updateElements(doc)
# If we are inserting, we shouldn't have any old elements, so don't
# bother removing them.
return ret
self.collection.replace_one = replaceElements
self.collection.insert_one = insertElements
result = super(Annotation, self).save(annotation, *args, **kwargs)
self.collection.replace_one = replace_one
self.collection.insert_one = insert_one
if _elementQuery:
result['_elementQuery'] = _elementQuery
logger.debug('Saved annotation in %5.3fs' % (time.time() - starttime))
return result
def finalizeInstance(event):
job = event.info['job']
if job.get("instance_id"):
instance = Instance().load(job["instance_id"], force=True)
if (instance["status"] == InstanceStatus.LAUNCHING
and job["status"] == JobStatus.ERROR # noqa
):
instance["status"] = InstanceStatus.ERROR
Instance().updateInstance(instance)
if job['title'] == 'Spawn Instance' and job.get('status') is not None:
status = int(job['status'])
instance_id = job['args'][0]['instanceId']
instance = Instance().load(instance_id, force=True, exc=True)
update = True
if (status == JobStatus.SUCCESS
and instance["status"] == InstanceStatus.LAUNCHING # noqa
):
service = getCeleryApp().AsyncResult(job['celeryTaskId']).get()
valid_keys = set(containerInfoSchema['properties'].keys())
containerInfo = {key: service.get(key, '') for key in valid_keys}
url = service.get('url', 'https://google.com')
_wait_for_server(url)
# Since _wait_for_server can potentially take some time,
# we need to refresh the state of the instance
instance = Instance().load(instance_id, force=True, exc=True)
if instance["status"] != InstanceStatus.LAUNCHING:
return # bail
# Preserve the imageId / current digest in containerInfo
tale = Tale().load(instance['taleId'], force=True)
containerInfo['imageId'] = tale['imageId']
containerInfo['digest'] = tale['imageInfo']['digest']
instance.update({
'url': url,
'status': InstanceStatus.RUNNING,
'containerInfo': containerInfo,
})
if "sessionId" in service:
instance["sessionId"] = ObjectId(service["sessionId"])
elif (status == JobStatus.ERROR
and instance["status"] != InstanceStatus.ERROR # noqa
):
instance['status'] = InstanceStatus.ERROR
elif (status in (JobStatus.QUEUED, JobStatus.RUNNING)
and instance["status"] != InstanceStatus.LAUNCHING # noqa
):
instance['status'] = InstanceStatus.LAUNCHING
else:
update = False
if update:
msg = "Updating instance ({_id}) in finalizeInstance".format(
**instance)
msg += " for job(id={_id}, status={status})".format(**job)
logger.debug(msg)
Instance().updateInstance(instance)
def yieldElements(self, annotation, region=None, info=None):
"""
Given an annotation, fetch the elements from the database and add them
to it.
When a region is used to request specific element, the following
keys can be specified:
left, right, top, bottom, low, high: the spatial area where
elements are located, all in pixels. If an element's bounding box is
at least partially within the requested area, that element is included.
minimumSize: the minimum size of an element to return.
sort, sortdir: standard sort options. The sort key can include
size and details.
limit: limit the total number of elements by this value. Defaults
to no limit.
offset: the offset within the query to start returning values. If
maxDetails is used, to get subsequent sets of elements, the offset
needs to be increased by the actual number of elements returned from a
previous query, which will vary based on the details of the elements.
maxDetails: if specified, limit the total number of elements by the
sum of their details values. This is applied in addition to limit.
The sum of the details values of the elements may exceed maxDetails
slightly (the sum of all but the last element will be less than
maxDetails, but the last element may exceed the value).
:param annotation: the annotation to get elements for. Modified.
:param region: if present, a dictionary restricting which annotations
are returned.
"""
info = info if info is not None else {}
region = region or {}
query = {
'annotationId': annotation.get('_annotationId', annotation['_id']),
'_version': annotation['_version']
}
for key in region:
if key in self.bboxKeys and self.bboxKeys[key][1]:
if self.bboxKeys[key][1] == '$gte' and float(region[key]) <= 0:
continue
query[self.bboxKeys[key][0]] = {
self.bboxKeys[key][1]: float(region[key])
}
if region.get('sort') in self.bboxKeys:
sortkey = self.bboxKeys[region['sort']][0]
else:
sortkey = region.get('sort') or '_id'
sortdir = int(
region['sortdir']) if region.get('sortdir') else SortDir.ASCENDING
limit = int(region['limit']) if region.get('limit') else 0
maxDetails = int(region.get('maxDetails') or 0)
queryLimit = maxDetails if maxDetails and (
not limit or maxDetails < limit) else limit
offset = int(region['offset']) if region.get('offset') else 0
logger.debug('element query %r for %r', query, region)
elementCursor = self.find(query=query,
sort=[(sortkey, sortdir)],
limit=queryLimit,
offset=offset,
fields={
'_id': True,
'element': True,
'bbox.details': True
})
info.update({
'count': elementCursor.count(),
'offset': offset,
'filter': query,
'sort': [sortkey, sortdir],
})
details = count = 0
if maxDetails:
info['maxDetails'] = maxDetails
if limit:
info['limit'] = limit
for entry in elementCursor:
element = entry['element']
element.setdefault('id', entry['_id'])
yield element
count += 1
details += entry.get('bbox', {}).get('details', 1)
if maxDetails and details >= maxDetails:
break
info['returned'] = count
info['details'] = details
def get_package_pid(path):
"""Get the pid of a package from its path."""
initial_pid = find_initial_pid(path)
logger.debug('Parsed initial PID of {}.'.format(initial_pid))
return find_resource_pid(initial_pid)
def yieldElements(self, annotation, region=None, info=None):
"""
Given an annotation, fetch the elements from the database.
When a region is used to request specific element, the following
keys can be specified:
:left, right, top, bottom, low, high: the spatial area where
elements are located, all in pixels. If an element's bounding
box is at least partially within the requested area, that
element is included.
:minimumSize: the minimum size of an element to return.
:sort, sortdir: standard sort options. The sort key can include
size and details.
:limit: limit the total number of elements by this value. Defaults
to no limit.
:offset: the offset within the query to start returning values. If
maxDetails is used, to get subsequent sets of elements, the
offset needs to be increased by the actual number of elements
returned from a previous query, which will vary based on the
details of the elements.
:maxDetails: if specified, limit the total number of elements by
the sum of their details values. This is applied in addition
to limit. The sum of the details values of the elements may
exceed maxDetails slightly (the sum of all but the last element
will be less than maxDetails, but the last element may exceed
the value).
:centroids: if specified and true, only return the id, center of
the bounding box, and bounding box size for each element.
:param annotation: the annotation to get elements for. Modified.
:param region: if present, a dictionary restricting which annotations
are returned.
:param info: an optional dictionary that will be modified with
additional query information, including count (total number of
available elements), returned (number of elements in response),
maxDetails (as specified by the region dictionary), details (sum of
details returned), limit (as specified by region), centroids (a
boolean based on the region specification).
:returns: a list of elements. If centroids were requested, each entry
is a list with str(id), x, y, size. Otherwise, each entry is the
element record.
"""
info = info if info is not None else {}
region = region or {}
query = {
'annotationId': annotation.get('_annotationId', annotation['_id']),
'_version': annotation['_version']
}
for key in region:
if key in self.bboxKeys and self.bboxKeys[key][1]:
if self.bboxKeys[key][1] == '$gte' and float(region[key]) <= 0:
continue
query[self.bboxKeys[key][0]] = {
self.bboxKeys[key][1]: float(region[key])
}
if region.get('sort') in self.bboxKeys:
sortkey = self.bboxKeys[region['sort']][0]
else:
sortkey = region.get('sort') or '_id'
sortdir = int(
region['sortdir']) if region.get('sortdir') else SortDir.ASCENDING
limit = int(region['limit']) if region.get('limit') else 0
maxDetails = int(region.get('maxDetails') or 0)
queryLimit = maxDetails if maxDetails and (
not limit or maxDetails < limit) else limit
offset = int(region['offset']) if region.get('offset') else 0
logger.debug('element query %r for %r', query, region)
fields = {'_id': True, 'element': True, 'bbox.details': True}
centroids = str(region.get('centroids')).lower() == 'true'
if centroids:
# fields = {'_id': True, 'element': True, 'bbox': True}
fields = {'_id': True, 'element.id': True, 'bbox': True}
proplist = []
propskeys = [
'type', 'fillColor', 'lineColor', 'lineWidth', 'closed'
]
for key in propskeys:
fields['element.%s' % key] = True
props = {}
info['centroids'] = True
info['props'] = proplist
info['propskeys'] = propskeys
elementCursor = self.find(query=query,
sort=[(sortkey, sortdir)],
limit=queryLimit,
offset=offset,
fields=fields)
info.update({
'count': elementCursor.count(),
'offset': offset,
'filter': query,
'sort': [sortkey, sortdir],
})
details = count = 0
if maxDetails:
info['maxDetails'] = maxDetails
if limit:
info['limit'] = limit
for entry in elementCursor:
element = entry['element']
element.setdefault('id', entry['_id'])
if centroids:
bbox = entry.get('bbox')
if not bbox or 'lowx' not in bbox or 'size' not in bbox:
continue
prop = tuple(element.get(key) for key in propskeys)
if prop not in props:
props[prop] = len(props)
proplist.append(list(prop))
yield [
str(element['id']), (bbox['lowx'] + bbox['highx']) / 2,
(bbox['lowy'] + bbox['highy']) / 2,
bbox['size'] if entry.get('type') != 'point' else 0,
props[prop]
]
details += 1
else:
yield element
details += entry.get('bbox', {}).get('details', 1)
count += 1
if maxDetails and details >= maxDetails:
break
info['returned'] = count
info['details'] = details
def addStyle(self, m, layerSrs, extent=None):
"""
Attaches raster style option to mapnik raster layer and adds the layer
to the mapnik map.
:param m: mapnik map.
:param layerSrs: the layer projection
:param extent: the extent to use for the mapnik layer.
"""
interpColorTable = {
'red': ['#000000', '#ff0000'],
'green': ['#000000', '#00ff00'],
'blue': ['#000000', '#0000ff'],
'gray': ['#000000', '#ffffff'],
'alpha': ['#ffffff00', '#ffffffff'],
}
bands = self.getBandInformation()
style = []
if hasattr(self, 'style'):
styleBands = self.style['bands'] if 'bands' in self.style else [self.style]
for styleBand in styleBands:
styleBand = styleBand.copy()
styleBand['band'] = self._bandNumber(styleBand.get('band'))
style.append(styleBand)
if not len(style):
for interp in ('red', 'green', 'blue', 'gray', 'palette', 'alpha'):
band = self._bandNumber(interp, False)
# If we don't have the requested band, or we only have alpha,
# or this is gray or palette and we already added another band,
# skip this interpretation.
if (band is None or
(interp == 'alpha' and not len(style)) or
(interp in ('gray', 'palette') and len(style))):
continue
if interp == 'palette':
style.append({'band': band, 'palette': 'colortable'})
else:
style.append({
'band': band,
'palette': interpColorTable[interp],
'min': 'auto',
'max': 'auto',
'nodata': 'auto',
'scheme': 'linear',
'composite': 'multiply' if interp == 'alpha' else 'lighten'
})
if not len(style):
style.append({'band': -1})
logger.debug('mapnik addTile specified style: %r, used style %r',
getattr(self, 'style', None), style)
for styleBand in style:
if styleBand['band'] != -1:
colorizer = self._colorizerFromStyle(styleBand)
composite = getattr(mapnik.CompositeOp, styleBand.get('composite', 'lighten'))
nodata = styleBand.get('nodata')
if nodata == 'auto':
nodata = bands.get('nodata')
else:
colorizer = None
composite = None
nodata = None
self._addStyleToMap(
m, layerSrs, colorizer, styleBand['band'], extent, composite, nodata)
def createCollection(self, name):
logger.debug('%s -> createCollection(%s)' % (self.getRefUrl(), name))
FolderResource.createCollection(self, name)
def save(self, annotation, *args, **kwargs):
"""
When saving an annotation, override the collection insert_one and
replace_one methods so that we don't save the elements with the main
annotation. Still use the super class's save method, so that all of
the triggers are fired as expected and cancelling and modifications can
be done as needed.
Because Mongo doesn't support transactions, a version number is stored
with the annotation and with the associated elements. This is used to
add the new elements first, then update the annotation, and delete the
old elements. The allows version integrity if another thread queries
the annotation at the same time.
:param annotation: the annotation document to save.
:returns: the saved document. If it is a new document, the _id has
been added.
"""
starttime = time.time()
with self._writeLock:
replace_one = self.collection.replace_one
insert_one = self.collection.insert_one
version = Annotationelement().getNextVersionValue()
if '_id' not in annotation:
oldversion = None
else:
if '_annotationId' in annotation:
annotation['_id'] = annotation['_annotationId']
# We read the old version from the existing record, because we
# don't want to trust that the input _version has not been altered
# or is present.
oldversion = self.collection.find_one(
{'_id': annotation['_id']}).get('_version')
annotation['_version'] = version
_elementQuery = annotation.pop('_elementQuery', None)
annotation.pop('_active', None)
annotation.pop('_annotationId', None)
def replaceElements(query, doc, *args, **kwargs):
Annotationelement().updateElements(doc)
elements = doc['annotation'].pop('elements', None)
if self._historyEnabled:
oldAnnotation = self.collection.find_one(query)
if oldAnnotation:
oldAnnotation['_annotationId'] = oldAnnotation.pop('_id')
oldAnnotation['_active'] = False
insert_one(oldAnnotation)
ret = replace_one(query, doc, *args, **kwargs)
if elements:
doc['annotation']['elements'] = elements
if not self._historyEnabled:
Annotationelement().removeOldElements(doc, oldversion)
return ret
def insertElements(doc, *args, **kwargs):
# When creating an annotation, store the elements first, then store
# the annotation without elements, then restore the elements.
doc.setdefault('_id', ObjectId())
if doc['annotation'].get('elements') is not None:
Annotationelement().updateElements(doc)
# If we are inserting, we shouldn't have any old elements, so don't
# bother removing them.
elements = doc['annotation'].pop('elements', None)
ret = insert_one(doc, *args, **kwargs)
if elements is not None:
doc['annotation']['elements'] = elements
return ret
with self._writeLock:
self.collection.replace_one = replaceElements
self.collection.insert_one = insertElements
try:
result = super(Annotation, self).save(annotation, *args, **kwargs)
finally:
self.collection.replace_one = replace_one
self.collection.insert_one = insert_one
if _elementQuery:
result['_elementQuery'] = _elementQuery
annotation.pop('groups', None)
self.injectAnnotationGroupSet(annotation)
logger.debug('Saved annotation in %5.3fs' % (time.time() - starttime))
events.trigger('large_image.annotations.save_history', {
'annotation': annotation
}, async=True)
return result
def genRESTEndPointsForSlicerCLIsInDockerCache(restResource, dockerCache):
"""Generates REST end points for slicer CLIs placed in subdirectories of a
given root directory and attaches them to a REST resource with the given
name.
For each CLI, it creates:
* a GET Route (<apiURL>/`restResourceName`/<cliRelativePath>/xmlspec)
that returns the xml spec of the CLI
* a POST Route (<apiURL>/`restResourceName`/<cliRelativePath>/run)
that runs the CLI
It also creates a GET route (<apiURL>/`restResourceName`) that returns a
list of relative routes to all CLIs attached to the generated REST resource
Parameters
----------
restResource : a dockerResource
REST resource to which the end-points should be attached
dockerCache : DockerCache object representing data stored in settings
"""
dockerImages = dockerCache.getImageNames()
# validate restResource argument
if not isinstance(restResource, Resource):
raise Exception('restResource must be a ' 'Docker Resource')
for dimg in dockerImages:
docker_image = dockerCache.getImageByName(dimg)
# get CLI list
cliListSpec = docker_image.getCLIListSpec()
# Add REST end-point for each CLI
for cliRelPath in cliListSpec.keys():
restPath = dimg.replace(':', '_').replace('/',
'_').replace('@', '_')
# create a POST REST route that runs the CLI
try:
cliXML = docker_image.getCLIXML(cliRelPath)
cliRunHandler = genHandlerToRunDockerCLI(
dimg, cliRelPath, cliXML, restResource)
except Exception:
logger.exception('Failed to create REST endpoints for %r',
cliRelPath)
continue
cliSuffix = os.path.normpath(cliRelPath).replace(os.sep, '_')
cliRunHandlerName = restPath + '_run_' + cliSuffix
setattr(restResource, cliRunHandlerName, cliRunHandler)
restResource.route('POST', (restPath, cliRelPath, 'run'),
getattr(restResource, cliRunHandlerName))
# store new rest endpoint
restResource.storeEndpoints(
dimg, cliRelPath, 'run',
['POST', (restPath, cliRelPath, 'run'), cliRunHandlerName])
# create GET REST route that returns the xml of the CLI
try:
cliGetXMLSpecHandler = genHandlerToGetDockerCLIXmlSpec(
cliRelPath, cliXML, restResource)
except Exception:
logger.exception('Failed to create REST endpoints for %s',
cliRelPath)
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error('%r', [exc_type, fname, exc_tb.tb_lineno])
continue
cliGetXMLSpecHandlerName = restPath + '_get_xml_' + cliSuffix
setattr(restResource, cliGetXMLSpecHandlerName,
cliGetXMLSpecHandler)
restResource.route('GET', (
restPath,
cliRelPath,
'xmlspec',
), getattr(restResource, cliGetXMLSpecHandlerName))
restResource.storeEndpoints(dimg, cliRelPath, 'xmlspec', [
'GET',
(restPath, cliRelPath, 'xmlspec'), cliGetXMLSpecHandlerName
])
logger.debug('Created REST endpoints for %s', cliRelPath)
return restResource
