def cache_tile_frames_job(job):
from girder_jobs.constants import JobStatus
from girder_jobs.models.job import Job
from girder_large_image.models.image_item import ImageItem
from girder import logger
kwargs = job['kwargs']
item = ImageItem().load(kwargs.pop('itemId'), force=True)
job = Job().updateJob(job,
log='Started caching tile frames\n',
status=JobStatus.RUNNING)
try:
for entry in kwargs.get('tileFramesList'):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.CANCELED:
return
job = Job().updateJob(job, log='Caching %r\n' % entry)
ImageItem().tileFrames(item, checkAndCreate=True, **entry)
job = Job().updateJob(job,
log='Finished caching tile frames\n',
status=JobStatus.SUCCESS)
except Exception as exc:
logger.exception('Failed caching tile frames')
job = Job().updateJob(job,
log='Failed caching tile frames (%s)\n' % exc,
status=JobStatus.ERROR)
def makeResources(server, fsAssetstore, admin, user):
# Create an item in the admin Public folder
adminPublicFolder = Folder().childFolders( # noqa: B305
admin, 'user', filters={
'name': 'Public'
}).next()
Item().createItem('Empty', admin, adminPublicFolder)
# Upload a sample file
file = utilities.uploadExternalFile(
'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-'
'af6f-14fefbbdf7bd.svs.sha512',
user,
fsAssetstore,
name='image')
item = Item().load(file['itemId'], force=True)
# We have to ask to make this a large image item, because we renamed it
# 'image' without an extension
ImageItem().createImageItem(item, file, user=user, createJob=False)
ImageItem().copyItem(item, user, name='copy')
annotation = Annotation().createAnnotation(item, admin,
{'name': 'admin annotation'})
annotation = Annotation().setAccessList(annotation, {},
force=True,
save=False)
annotation = Annotation().setPublic(annotation, True, save=True)
def _overlayBounds(self, overlayElement):
"""
Compute bounding box information in the X-Y plane for an
image overlay element.
This uses numpy to perform the specified transform on the given girder
image item in order to obtain bounding box coordinates.
:param overlayElement: An annotation element of type 'image'.
:returns: a tuple with 4 values: lowx, highx, lowy, highy. Runtime exceptions
during loading the image metadata will result in the tuple (0, 0, 0, 0).
"""
if overlayElement.get('type') not in ['image', 'pixelmap']:
raise ValueError(
'Function _overlayBounds only accepts annotation elements of type "image", '
'"pixelmap."')
import numpy as np
lowx = highx = lowy = highy = 0
try:
overlayItemId = overlayElement.get('girderId')
imageItem = ImageItem().load(overlayItemId, force=True)
overlayImageMetadata = ImageItem().getMetadata(imageItem)
corners = [
[0, 0],
[0, overlayImageMetadata['sizeY']],
[overlayImageMetadata['sizeX'], overlayImageMetadata['sizeY']],
[overlayImageMetadata['sizeX'], 0],
]
transform = overlayElement.get('transform', {})
transformMatrix = np.array(
transform.get('matrix', [[1, 0], [0, 1]]))
corners = [
np.matmul(np.array(corner), transformMatrix)
for corner in corners
]
offsetArray = np.array(
[transform.get('xoffset', 0),
transform.get('yoffset', 0)])
corners = [np.add(corner, offsetArray) for corner in corners]
# use .item() to convert back to native python types
lowx = min([corner[0] for corner in corners]).item()
highx = max([corner[0] for corner in corners]).item()
lowy = min([corner[1] for corner in corners]).item()
highy = max([corner[1] for corner in corners]).item()
except Exception:
logger.exception(
'Error generating bounding box for image overlay annotation')
return lowx, highx, lowy, highy
def resolveAnnotationGirderIds(event, results, data, possibleGirderIds):
"""
If an annotation has references to girderIds, resolve them to actual ids.
:param event: a data.process event.
:param results: the results from _itemFromEvent,
:param data: annotation data.
:param possibleGirderIds: a list of annotation elements with girderIds
needing resolution.
:returns: True if all ids were processed.
"""
# Exclude actual girderIds from resolution
girderIds = []
for element in possibleGirderIds:
# This will throw an exception if the girderId isn't well-formed as an
# actual id.
try:
if Item().load(element['girderId'],
level=AccessType.READ,
force=True) is None:
girderIds.append(element)
except Exception:
girderIds.append(element)
if not len(girderIds):
return True
idRecord = _recentIdentifiers.get(results.get('uuid'))
if idRecord and not all(element['girderId'] in idRecord
for element in girderIds):
idRecord['_reprocess'] = lambda: process_annotations(event)
return False
for element in girderIds:
element['girderId'] = str(
idRecord[element['girderId']]['file']['itemId'])
# Currently, all girderIds inside annotations are expected to be
# large images. In this case, load them and ask if they can be so,
# in case they are small images
from girder_large_image.models.image_item import ImageItem
try:
item = ImageItem().load(element['girderId'], force=True)
ImageItem().createImageItem(item,
list(ImageItem().childFiles(
item=item, limit=1))[0],
createJob=False)
except Exception:
pass
return True
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'):
"""
As needed, download sample data.
:param adminUser: a user to create and modify collections and folders.
:param collName: the collection name where the data will be added.
:param folderName: the folder name where the data will bed aded.
:returns: the folder where the sample data is located.
"""
try:
import girder_client
except ImportError:
logger.error('girder_client is unavailable. Cannot get sample data.')
return
from girder.models.item import Item
from girder.models.upload import Upload
from girder_large_image.models.image_item import ImageItem
folder = get_collection_folder(adminUser, collName, folderName)
remote = girder_client.GirderClient(
apiUrl='https://data.kitware.com/api/v1')
remoteFolder = remote.resourceLookup(
'/collection/HistomicsTK/Deployment test images')
sampleItems = []
for remoteItem in remote.listItem(remoteFolder['_id']):
item = Item().findOne({
'folderId': folder['_id'],
'name': remoteItem['name']
})
if item and len(list(Item().childFiles(item, limit=1))):
sampleItems.append(item)
continue
if not item:
item = Item().createItem(remoteItem['name'],
creator=adminUser,
folder=folder)
for remoteFile in remote.listFile(remoteItem['_id']):
with tempfile.NamedTemporaryFile() as tf:
fileName = tf.name
tf.close()
logger.info('Downloading %s', remoteFile['name'])
remote.downloadFile(remoteFile['_id'], fileName)
Upload().uploadFromFile(open(fileName, 'rb'),
os.path.getsize(fileName),
name=remoteItem['name'],
parentType='item',
parent=item,
user=adminUser)
sampleItems.append(item)
for item in sampleItems:
if 'largeImage' not in item:
logger.info('Making large_item %s', item['name'])
try:
ImageItem().createImageItem(item, createJob=False)
except Exception:
pass
logger.info('done')
return folder
def findAnnotatedImages(self,
imageNameFilter=None,
creator=None,
user=None,
level=AccessType.ADMIN,
force=None,
offset=0,
limit=0,
sort=None,
**kwargs):
r"""
Find images associated with annotations.
The list returned by this function is paginated and filtered by access control using
the standard girder kwargs.
:param imageNameFilter: A string used to filter images by name. An image name matches
if it (or a subtoken) begins with this string. Subtokens are generated by splitting
by the regex ``[\W_]+`` This filter is case-insensitive.
:param creator: Filter by a user who is the creator of the annotation.
"""
query = {'_active': {'$ne': False}}
if creator:
query['creatorId'] = creator['_id']
annotations = self.find(query, sort=sort, fields=['itemId'])
images = []
imageIds = set()
for annotation in annotations:
# short cut if the image has already been added to the results
if annotation['itemId'] in imageIds:
continue
try:
item = ImageItem().load(annotation['itemId'],
level=level,
user=user,
force=force)
except AccessException:
item = None
# ignore if no such item exists
if not item:
continue
if not self._matchImageName(item['name'], imageNameFilter or ''):
continue
if len(imageIds) >= offset:
images.append(item)
imageIds.add(item['_id'])
if len(images) == limit:
break
return images
def _resolveSourcePath(self, sources, source):
try:
super()._resolveSourcePath(sources, source)
except TileSourceFileNotFoundError:
prefix = 'girder://'
potentialId = source['path']
if potentialId.startswith(prefix):
potentialId = potentialId[len(prefix):]
if '://' not in potentialId:
try:
item = ImageItem().load(potentialId, force=True)
ts = ImageItem().tileSource(item)
source = copy.deepcopy(source)
source['path'] = ts._getLargeImagePath()
source['sourceName'] = item['largeImage']['sourceName']
sources.append(source)
return
except Exception:
pass
raise
def _setLargeImage(self, doc, fileId, user, token):
if doc.get('largeImage', {}).get('fileId') == fileId:
return
file = File().load(fileId, user=user)
try:
return ImageItem.createImageItem(doc,
file,
user=user,
token=token,
createJob=False)
except Exception:
raise ValidationException('Could not generate a large_image item')
def thumbnail(self, image, params):
width = int(params.get('width', 256))
height = int(params.get('height', 256))
thumbData, thumbMime = ImageItem().getThumbnail(image,
width=width,
height=height)
# Only setRawResponse now, as this handler may return a JSON error
# earlier
setRawResponse()
setResponseHeader('Content-Type', thumbMime)
return thumbData
def getTile(self, image, z, x, y, params):
try:
x, y, z = int(x), int(y), int(z)
except ValueError:
raise RestException('x, y, and z must be integers')
if x < 0 or y < 0 or z < 0:
raise RestException('x, y, and z must be positive integers')
try:
tileData, tileMime = ImageItem().getTile(image, x, y, z)
except TileGeneralException as e:
raise RestException(str(e), code=404)
setResponseHeader('Content-Type', tileMime)
setRawResponse()
return tileData
def testGetLargeImagePath(server, admin, fsAssetstore):
file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
itemId = str(file['itemId'])
item = Item().load(itemId, user=admin)
ts = ImageItem().tileSource(item)
with mock.patch.object(File(), 'getGirderMountFilePath', return_value='mockmount'):
path = ts._getLargeImagePath()
abspath = os.path.abspath(path)
assert path != file['path']
assert path.endswith(file['path'])
ts._mayHaveAdjacentFiles = True
path = ts._getLargeImagePath()
assert path == 'mockmount'
origFile = file
file['imported'] = True
file['path'] = abspath
file = File().save(file)
path = ts._getLargeImagePath()
assert path == abspath
file = File().save(origFile)
def testThumbnailFileJob(server, admin, user, fsAssetstore):
file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
itemId = str(file['itemId'])
# We should report zero thumbnails
item = Item().load(itemId, user=admin)
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 0
# Test PUT thumbnails
resp = server.request(method='PUT', path='/large_image/thumbnails', user=user)
assert utilities.respStatus(resp) == 403
resp = server.request(method='PUT', path='/large_image/thumbnails', user=admin)
assert utilities.respStatus(resp) == 400
assert '"spec" is required' in resp.json['message']
resp = server.request(
method='PUT', path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps({})})
assert utilities.respStatus(resp) == 400
assert 'must be a JSON list' in resp.json['message']
# Run a job to create two sizes of thumbnails
assert _createThumbnails(server, admin, [
{'width': 160, 'height': 100},
{'encoding': 'PNG'}
])
# We should report two thumbnails
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 2
# Run a job to create two sizes of thumbnails, one different than
# before
assert _createThumbnails(server, admin, [
{'width': 160, 'height': 100},
{'width': 160, 'height': 160},
])
# We should report three thumbnails
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 3
# Asking for a bad thumbnail specification should just do nothing
assert not _createThumbnails(server, admin, ['not a dictionary'])
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 3
# Test GET thumbnails
resp = server.request(path='/large_image/thumbnails', user=user)
assert utilities.respStatus(resp) == 403
resp = server.request(
path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps({})})
assert utilities.respStatus(resp) == 400
assert 'must be a JSON list' in resp.json['message']
resp = server.request(path='/large_image/thumbnails', user=admin)
assert utilities.respStatus(resp) == 200
assert resp.json == 3
resp = server.request(
path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps([{'width': 160, 'height': 100}])})
assert utilities.respStatus(resp) == 200
assert resp.json == 1
# Test DELETE thumbnails
resp = server.request(method='DELETE', path='/large_image/thumbnails', user=user)
assert utilities.respStatus(resp) == 403
resp = server.request(
method='DELETE', path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps({})})
assert utilities.respStatus(resp) == 400
assert 'must be a JSON list' in resp.json['message']
# Delete one set of thumbnails
resp = server.request(
method='DELETE', path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps([{'encoding': 'PNG'}])})
assert utilities.respStatus(resp) == 200
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 2
# Try to delete some that don't exist
resp = server.request(
method='DELETE', path='/large_image/thumbnails', user=admin,
params={'spec': json.dumps([{'width': 200, 'height': 200}])})
assert utilities.respStatus(resp) == 200
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 2
# Delete them all
resp = server.request(
method='DELETE', path='/large_image/thumbnails', user=admin)
assert utilities.respStatus(resp) == 200
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present == 0
# We should be able to cancel a job
slowList = [
{'width': 1600, 'height': 1000},
{'width': 3200, 'height': 2000},
{'width': 1600, 'height': 1002},
{'width': 1600, 'height': 1003},
{'width': 1600, 'height': 1004},
]
assert _createThumbnails(server, admin, slowList, cancel=True) == 'canceled'
present, removed = ImageItem().removeThumbnailFiles(item, keep=10)
assert present < 3 + len(slowList)
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'):
"""
As needed, download sample data.
:param adminUser: a user to create and modify collections and folders.
:param collName: the collection name where the data will be added.
:param folderName: the folder name where the data will bed aded.
:returns: the folder where the sample data is located.
"""
if Collection().findOne({'lowerName': collName.lower()}) is None:
Collection().createCollection(collName, adminUser)
collection = Collection().findOne({'lowerName': collName.lower()})
if Folder().findOne({
'parentId': collection['_id'],
'lowerName': folderName.lower()
}) is None:
Folder().createFolder(collection,
folderName,
parentType='collection',
public=True,
creator=adminUser)
folder = Folder().findOne({
'parentId': collection['_id'],
'lowerName': folderName.lower()
})
remote = girder_client.GirderClient(
apiUrl='https://data.kitware.com/api/v1')
remoteFolder = remote.resourceLookup(
'/collection/HistomicsTK/Deployment test images')
sampleItems = []
for remoteItem in remote.listItem(remoteFolder['_id']):
item = Item().findOne({
'folderId': folder['_id'],
'name': remoteItem['name']
})
if item and len(list(Item().childFiles(item, limit=1))):
sampleItems.append(item)
continue
if not item:
item = Item().createItem(remoteItem['name'],
creator=adminUser,
folder=folder)
for remoteFile in remote.listFile(remoteItem['_id']):
with tempfile.NamedTemporaryFile() as tf:
fileName = tf.name
tf.close()
sys.stdout.write('Downloading %s' % remoteFile['name'])
sys.stdout.flush()
remote.downloadFile(remoteFile['_id'], fileName)
sys.stdout.write(' .')
sys.stdout.flush()
Upload().uploadFromFile(open(fileName, 'rb'),
os.path.getsize(fileName),
name=remoteItem['name'],
parentType='item',
parent=item,
user=adminUser)
sys.stdout.write('.\n')
sys.stdout.flush()
sampleItems.append(item)
for item in sampleItems:
if 'largeImage' not in item:
sys.stdout.write('Making large_item %s ' % item['name'])
sys.stdout.flush()
try:
ImageItem().createImageItem(item, createJob=False)
except Exception:
pass
print('done')
return folder
def getTileInfo(self, image, params):
# These endpoints should guarantee that large_image functionality works, so a
# TileGeneralException can be treated as an internal server error and not get caught
return ImageItem().getMetadata(image)