def cache_tile_frames_job(job): from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job from girder_large_image.models.image_item import ImageItem from girder import logger kwargs = job['kwargs'] item = ImageItem().load(kwargs.pop('itemId'), force=True) job = Job().updateJob(job, log='Started caching tile frames\n', status=JobStatus.RUNNING) try: for entry in kwargs.get('tileFramesList'): job = Job().load(job['_id'], force=True) if job['status'] == JobStatus.CANCELED: return job = Job().updateJob(job, log='Caching %r\n' % entry) ImageItem().tileFrames(item, checkAndCreate=True, **entry) job = Job().updateJob(job, log='Finished caching tile frames\n', status=JobStatus.SUCCESS) except Exception as exc: logger.exception('Failed caching tile frames') job = Job().updateJob(job, log='Failed caching tile frames (%s)\n' % exc, status=JobStatus.ERROR)
def makeResources(server, fsAssetstore, admin, user): # Create an item in the admin Public folder adminPublicFolder = Folder().childFolders( # noqa: B305 admin, 'user', filters={ 'name': 'Public' }).next() Item().createItem('Empty', admin, adminPublicFolder) # Upload a sample file file = utilities.uploadExternalFile( 'data/sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-' 'af6f-14fefbbdf7bd.svs.sha512', user, fsAssetstore, name='image') item = Item().load(file['itemId'], force=True) # We have to ask to make this a large image item, because we renamed it # 'image' without an extension ImageItem().createImageItem(item, file, user=user, createJob=False) ImageItem().copyItem(item, user, name='copy') annotation = Annotation().createAnnotation(item, admin, {'name': 'admin annotation'}) annotation = Annotation().setAccessList(annotation, {}, force=True, save=False) annotation = Annotation().setPublic(annotation, True, save=True)
def _overlayBounds(self, overlayElement): """ Compute bounding box information in the X-Y plane for an image overlay element. This uses numpy to perform the specified transform on the given girder image item in order to obtain bounding box coordinates. :param overlayElement: An annotation element of type 'image'. :returns: a tuple with 4 values: lowx, highx, lowy, highy. Runtime exceptions during loading the image metadata will result in the tuple (0, 0, 0, 0). """ if overlayElement.get('type') not in ['image', 'pixelmap']: raise ValueError( 'Function _overlayBounds only accepts annotation elements of type "image", ' '"pixelmap."') import numpy as np lowx = highx = lowy = highy = 0 try: overlayItemId = overlayElement.get('girderId') imageItem = ImageItem().load(overlayItemId, force=True) overlayImageMetadata = ImageItem().getMetadata(imageItem) corners = [ [0, 0], [0, overlayImageMetadata['sizeY']], [overlayImageMetadata['sizeX'], overlayImageMetadata['sizeY']], [overlayImageMetadata['sizeX'], 0], ] transform = overlayElement.get('transform', {}) transformMatrix = np.array( transform.get('matrix', [[1, 0], [0, 1]])) corners = [ np.matmul(np.array(corner), transformMatrix) for corner in corners ] offsetArray = np.array( [transform.get('xoffset', 0), transform.get('yoffset', 0)]) corners = [np.add(corner, offsetArray) for corner in corners] # use .item() to convert back to native python types lowx = min([corner[0] for corner in corners]).item() highx = max([corner[0] for corner in corners]).item() lowy = min([corner[1] for corner in corners]).item() highy = max([corner[1] for corner in corners]).item() except Exception: logger.exception( 'Error generating bounding box for image overlay annotation') return lowx, highx, lowy, highy
def resolveAnnotationGirderIds(event, results, data, possibleGirderIds): """ If an annotation has references to girderIds, resolve them to actual ids. :param event: a data.process event. :param results: the results from _itemFromEvent, :param data: annotation data. :param possibleGirderIds: a list of annotation elements with girderIds needing resolution. :returns: True if all ids were processed. """ # Exclude actual girderIds from resolution girderIds = [] for element in possibleGirderIds: # This will throw an exception if the girderId isn't well-formed as an # actual id. try: if Item().load(element['girderId'], level=AccessType.READ, force=True) is None: girderIds.append(element) except Exception: girderIds.append(element) if not len(girderIds): return True idRecord = _recentIdentifiers.get(results.get('uuid')) if idRecord and not all(element['girderId'] in idRecord for element in girderIds): idRecord['_reprocess'] = lambda: process_annotations(event) return False for element in girderIds: element['girderId'] = str( idRecord[element['girderId']]['file']['itemId']) # Currently, all girderIds inside annotations are expected to be # large images. In this case, load them and ask if they can be so, # in case they are small images from girder_large_image.models.image_item import ImageItem try: item = ImageItem().load(element['girderId'], force=True) ImageItem().createImageItem(item, list(ImageItem().childFiles( item=item, limit=1))[0], createJob=False) except Exception: pass return True
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'): """ As needed, download sample data. :param adminUser: a user to create and modify collections and folders. :param collName: the collection name where the data will be added. :param folderName: the folder name where the data will bed aded. :returns: the folder where the sample data is located. """ try: import girder_client except ImportError: logger.error('girder_client is unavailable. Cannot get sample data.') return from girder.models.item import Item from girder.models.upload import Upload from girder_large_image.models.image_item import ImageItem folder = get_collection_folder(adminUser, collName, folderName) remote = girder_client.GirderClient( apiUrl='https://data.kitware.com/api/v1') remoteFolder = remote.resourceLookup( '/collection/HistomicsTK/Deployment test images') sampleItems = [] for remoteItem in remote.listItem(remoteFolder['_id']): item = Item().findOne({ 'folderId': folder['_id'], 'name': remoteItem['name'] }) if item and len(list(Item().childFiles(item, limit=1))): sampleItems.append(item) continue if not item: item = Item().createItem(remoteItem['name'], creator=adminUser, folder=folder) for remoteFile in remote.listFile(remoteItem['_id']): with tempfile.NamedTemporaryFile() as tf: fileName = tf.name tf.close() logger.info('Downloading %s', remoteFile['name']) remote.downloadFile(remoteFile['_id'], fileName) Upload().uploadFromFile(open(fileName, 'rb'), os.path.getsize(fileName), name=remoteItem['name'], parentType='item', parent=item, user=adminUser) sampleItems.append(item) for item in sampleItems: if 'largeImage' not in item: logger.info('Making large_item %s', item['name']) try: ImageItem().createImageItem(item, createJob=False) except Exception: pass logger.info('done') return folder
def findAnnotatedImages(self, imageNameFilter=None, creator=None, user=None, level=AccessType.ADMIN, force=None, offset=0, limit=0, sort=None, **kwargs): r""" Find images associated with annotations. The list returned by this function is paginated and filtered by access control using the standard girder kwargs. :param imageNameFilter: A string used to filter images by name. An image name matches if it (or a subtoken) begins with this string. Subtokens are generated by splitting by the regex ``[\W_]+`` This filter is case-insensitive. :param creator: Filter by a user who is the creator of the annotation. """ query = {'_active': {'$ne': False}} if creator: query['creatorId'] = creator['_id'] annotations = self.find(query, sort=sort, fields=['itemId']) images = [] imageIds = set() for annotation in annotations: # short cut if the image has already been added to the results if annotation['itemId'] in imageIds: continue try: item = ImageItem().load(annotation['itemId'], level=level, user=user, force=force) except AccessException: item = None # ignore if no such item exists if not item: continue if not self._matchImageName(item['name'], imageNameFilter or ''): continue if len(imageIds) >= offset: images.append(item) imageIds.add(item['_id']) if len(images) == limit: break return images
def _resolveSourcePath(self, sources, source): try: super()._resolveSourcePath(sources, source) except TileSourceFileNotFoundError: prefix = 'girder://' potentialId = source['path'] if potentialId.startswith(prefix): potentialId = potentialId[len(prefix):] if '://' not in potentialId: try: item = ImageItem().load(potentialId, force=True) ts = ImageItem().tileSource(item) source = copy.deepcopy(source) source['path'] = ts._getLargeImagePath() source['sourceName'] = item['largeImage']['sourceName'] sources.append(source) return except Exception: pass raise
def _setLargeImage(self, doc, fileId, user, token): if doc.get('largeImage', {}).get('fileId') == fileId: return file = File().load(fileId, user=user) try: return ImageItem.createImageItem(doc, file, user=user, token=token, createJob=False) except Exception: raise ValidationException('Could not generate a large_image item')
def thumbnail(self, image, params): width = int(params.get('width', 256)) height = int(params.get('height', 256)) thumbData, thumbMime = ImageItem().getThumbnail(image, width=width, height=height) # Only setRawResponse now, as this handler may return a JSON error # earlier setRawResponse() setResponseHeader('Content-Type', thumbMime) return thumbData
def getTile(self, image, z, x, y, params): try: x, y, z = int(x), int(y), int(z) except ValueError: raise RestException('x, y, and z must be integers') if x < 0 or y < 0 or z < 0: raise RestException('x, y, and z must be positive integers') try: tileData, tileMime = ImageItem().getTile(image, x, y, z) except TileGeneralException as e: raise RestException(str(e), code=404) setResponseHeader('Content-Type', tileMime) setRawResponse() return tileData
def testGetLargeImagePath(server, admin, fsAssetstore): file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore) itemId = str(file['itemId']) item = Item().load(itemId, user=admin) ts = ImageItem().tileSource(item) with mock.patch.object(File(), 'getGirderMountFilePath', return_value='mockmount'): path = ts._getLargeImagePath() abspath = os.path.abspath(path) assert path != file['path'] assert path.endswith(file['path']) ts._mayHaveAdjacentFiles = True path = ts._getLargeImagePath() assert path == 'mockmount' origFile = file file['imported'] = True file['path'] = abspath file = File().save(file) path = ts._getLargeImagePath() assert path == abspath file = File().save(origFile)
def testThumbnailFileJob(server, admin, user, fsAssetstore): file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore) itemId = str(file['itemId']) # We should report zero thumbnails item = Item().load(itemId, user=admin) present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 0 # Test PUT thumbnails resp = server.request(method='PUT', path='/large_image/thumbnails', user=user) assert utilities.respStatus(resp) == 403 resp = server.request(method='PUT', path='/large_image/thumbnails', user=admin) assert utilities.respStatus(resp) == 400 assert '"spec" is required' in resp.json['message'] resp = server.request( method='PUT', path='/large_image/thumbnails', user=admin, params={'spec': json.dumps({})}) assert utilities.respStatus(resp) == 400 assert 'must be a JSON list' in resp.json['message'] # Run a job to create two sizes of thumbnails assert _createThumbnails(server, admin, [ {'width': 160, 'height': 100}, {'encoding': 'PNG'} ]) # We should report two thumbnails present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 2 # Run a job to create two sizes of thumbnails, one different than # before assert _createThumbnails(server, admin, [ {'width': 160, 'height': 100}, {'width': 160, 'height': 160}, ]) # We should report three thumbnails present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 3 # Asking for a bad thumbnail specification should just do nothing assert not _createThumbnails(server, admin, ['not a dictionary']) present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 3 # Test GET thumbnails resp = server.request(path='/large_image/thumbnails', user=user) assert utilities.respStatus(resp) == 403 resp = server.request( path='/large_image/thumbnails', user=admin, params={'spec': json.dumps({})}) assert utilities.respStatus(resp) == 400 assert 'must be a JSON list' in resp.json['message'] resp = server.request(path='/large_image/thumbnails', user=admin) assert utilities.respStatus(resp) == 200 assert resp.json == 3 resp = server.request( path='/large_image/thumbnails', user=admin, params={'spec': json.dumps([{'width': 160, 'height': 100}])}) assert utilities.respStatus(resp) == 200 assert resp.json == 1 # Test DELETE thumbnails resp = server.request(method='DELETE', path='/large_image/thumbnails', user=user) assert utilities.respStatus(resp) == 403 resp = server.request( method='DELETE', path='/large_image/thumbnails', user=admin, params={'spec': json.dumps({})}) assert utilities.respStatus(resp) == 400 assert 'must be a JSON list' in resp.json['message'] # Delete one set of thumbnails resp = server.request( method='DELETE', path='/large_image/thumbnails', user=admin, params={'spec': json.dumps([{'encoding': 'PNG'}])}) assert utilities.respStatus(resp) == 200 present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 2 # Try to delete some that don't exist resp = server.request( method='DELETE', path='/large_image/thumbnails', user=admin, params={'spec': json.dumps([{'width': 200, 'height': 200}])}) assert utilities.respStatus(resp) == 200 present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 2 # Delete them all resp = server.request( method='DELETE', path='/large_image/thumbnails', user=admin) assert utilities.respStatus(resp) == 200 present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present == 0 # We should be able to cancel a job slowList = [ {'width': 1600, 'height': 1000}, {'width': 3200, 'height': 2000}, {'width': 1600, 'height': 1002}, {'width': 1600, 'height': 1003}, {'width': 1600, 'height': 1004}, ] assert _createThumbnails(server, admin, slowList, cancel=True) == 'canceled' present, removed = ImageItem().removeThumbnailFiles(item, keep=10) assert present < 3 + len(slowList)
def get_sample_data(adminUser, collName='Sample Images', folderName='Images'): """ As needed, download sample data. :param adminUser: a user to create and modify collections and folders. :param collName: the collection name where the data will be added. :param folderName: the folder name where the data will bed aded. :returns: the folder where the sample data is located. """ if Collection().findOne({'lowerName': collName.lower()}) is None: Collection().createCollection(collName, adminUser) collection = Collection().findOne({'lowerName': collName.lower()}) if Folder().findOne({ 'parentId': collection['_id'], 'lowerName': folderName.lower() }) is None: Folder().createFolder(collection, folderName, parentType='collection', public=True, creator=adminUser) folder = Folder().findOne({ 'parentId': collection['_id'], 'lowerName': folderName.lower() }) remote = girder_client.GirderClient( apiUrl='https://data.kitware.com/api/v1') remoteFolder = remote.resourceLookup( '/collection/HistomicsTK/Deployment test images') sampleItems = [] for remoteItem in remote.listItem(remoteFolder['_id']): item = Item().findOne({ 'folderId': folder['_id'], 'name': remoteItem['name'] }) if item and len(list(Item().childFiles(item, limit=1))): sampleItems.append(item) continue if not item: item = Item().createItem(remoteItem['name'], creator=adminUser, folder=folder) for remoteFile in remote.listFile(remoteItem['_id']): with tempfile.NamedTemporaryFile() as tf: fileName = tf.name tf.close() sys.stdout.write('Downloading %s' % remoteFile['name']) sys.stdout.flush() remote.downloadFile(remoteFile['_id'], fileName) sys.stdout.write(' .') sys.stdout.flush() Upload().uploadFromFile(open(fileName, 'rb'), os.path.getsize(fileName), name=remoteItem['name'], parentType='item', parent=item, user=adminUser) sys.stdout.write('.\n') sys.stdout.flush() sampleItems.append(item) for item in sampleItems: if 'largeImage' not in item: sys.stdout.write('Making large_item %s ' % item['name']) sys.stdout.flush() try: ImageItem().createImageItem(item, createJob=False) except Exception: pass print('done') return folder
def getTileInfo(self, image, params): # These endpoints should guarantee that large_image functionality works, so a # TileGeneralException can be treated as an internal server error and not get caught return ImageItem().getMetadata(image)