def tile(mode, project, specimen, block, resource, slide_name, slide_ext,
level, col, row, format):
format = format.lower()
if format != 'jpeg' and format != 'png':
# Not supported by Deep Zoom
return 'bad format'
abort(404)
# Get a project reference, using either local database or remotely supplied dict
pr = dzi_get_project_ref(project)
# Get the raw SVS/tiff file for the slide (the resource should exist,
# or else we will spend a minute here waiting with no response to user)
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
tiff_file = sr.get_local_copy(resource)
os = None
if mode == 'affine':
os = AffineTransformedOpenSlide(
tiff_file, get_affine_matrix(sr, 'affine', resource, 'image'))
else:
os = OpenSlide(tiff_file)
dz = DeepZoomGenerator(os)
tile = dz.get_tile(level, (col, row))
buf = PILBytesIO()
tile.save(buf, format, quality=75)
resp = make_response(buf.getvalue())
resp.mimetype = 'image/%s' % format
print('PNG size: %d' % (len(buf.getvalue(), )))
return resp
def get_random_patch(project, specimen, block, resource, slide_name, slide_ext,
level, w, format):
format = format.lower()
if format != 'jpeg' and format != 'png':
# Not supported by Deep Zoom
return 'bad format'
# Get a project reference, using either local database or remotely supplied dict
pr = dzi_get_project_ref(project)
# Get the raw SVS/tiff file for the slide (the resource should exist,
# or else we will spend a minute here waiting with no response to user)
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
tiff_file = sr.get_local_copy(resource)
# Read the region centered on the box of size 512x512
os = OpenSlide(tiff_file)
# Work out the offset
cx = randint(
0,
int(os.level_dimensions[level][0] - w * os.level_downsamples[level]))
cy = randint(
0,
int(os.level_dimensions[level][1] - w * os.level_downsamples[level]))
tile = os.read_region((cx, cy), level, (w, w))
# Convert to PNG
buf = PILBytesIO()
tile.save(buf, format)
resp = make_response(buf.getvalue())
resp.mimetype = 'image/%s' % format
return resp
def dzi_preload(project, specimen, block, resource, slide_name, slide_ext):
# Get a project reference, using either local database or remotely supplied dict
pr = dzi_get_project_ref(project)
# Check if the file exists locally. If so, there is no need to queue a worker
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
tiff_file = sr.get_local_copy(resource, check_hash=True, dry_run=True)
if tiff_file is not None:
return json.dumps({"status": JobStatus.FINISHED})
# Get a redis queue
q = Queue(current_app.config['PRELOAD_QUEUE'], connection=Redis())
job = q.enqueue(do_preload_file,
project,
pr.get_dict(),
specimen,
block,
resource,
slide_name,
slide_ext,
job_timeout="300s",
result_ttl="60s")
# Stick the properties into the job
job.meta['args'] = (project, specimen, block, resource, slide_name,
slide_ext)
job.save_meta()
# Return the job id
return json.dumps({"job_id": job.id, "status": JobStatus.QUEUED})
def do_preload_file(project, proj_dict, specimen, block, resource, slide_name,
slide_ext):
sr = SlideRef(ProjectRef(project, proj_dict), specimen, block, slide_name,
slide_ext)
print('Fetching %s %s %s %s.%s' %
(project, specimen, block, slide_name, slide_ext))
tiff_file = sr.get_local_copy(resource, check_hash=True)
print('Fetched to %s' % tiff_file)
return tiff_file
def dzi_download(mode, project, specimen, block, resource, slide_name,
slide_ext):
# Get a project reference, using either local database or remotely supplied dict
pr = dzi_get_project_ref(project)
# Get the raw SVS/tiff file for the slide (the resource should exist,
# or else we will spend a minute here waiting with no response to user)
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
# Get the resource
tiff_file = sr.get_local_copy(resource, check_hash=True)
return send_file(tiff_file)
def dzi_job_status(project, slide_name, job_id):
pr = dzi_get_project_ref(project)
q = Queue(current_app.config['PRELOAD_QUEUE'], connection=Redis())
j = q.fetch_job(job_id)
if j is None:
return 'bad job'
abort(404)
res = {'status': j.get_status()}
print('JOB status: ', j)
if j.get_status() == JobStatus.STARTED:
(project, specimen, block, resource, slide_name,
slide_ext) = j.meta['args']
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
res['progress'] = sr.get_download_progress(resource)
return json.dumps(res)
def dzi(mode, project, specimen, block, resource, slide_name, slide_ext):
# Get a project reference, using either local database or remotely supplied dict
pr = dzi_get_project_ref(project)
# Get the raw SVS/tiff file for the slide (the resource should exist,
# or else we will spend a minute here waiting with no response to user)
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
tiff_file = sr.get_local_copy(resource, check_hash=True)
# Get an affine transform if that is an option
A = get_affine_matrix(sr, mode, resource, 'image')
# Load the slide
osa = AffineTransformedOpenSlide(tiff_file, A)
dz = DeepZoomGenerator(osa)
dz.filename = os.path.basename(tiff_file)
resp = make_response(dz.get_dzi('png'))
resp.mimetype = 'application/xml'
return resp
def slide_view(task_id, slide_id, resolution, affine_mode):
# Get the current task data
project, task = get_task_data(task_id)
# Get the next/previous slides for this task
si, prev_slide, next_slide, stain_list, user_prefs = get_slide_info(
task_id, slide_id)
# Check that the affine mode and resolution requested are available
pr = ProjectRef(project)
sr = SlideRef(pr, si['specimen_name'], si['block_name'], si['slide_name'],
si['slide_ext'])
have_affine = sr.resource_exists('affine', True) or sr.resource_exists(
'affine', False)
have_x16 = sr.resource_exists('x16', True) or sr.resource_exists(
'x16', False)
# If one is missing, we need a redirect
rd_affine_mode = affine_mode if have_affine else 'raw'
rd_resolution = resolution if have_x16 else 'raw'
# Get the list of available overlays and jsonify
overlays = sr.get_available_overlays(local=False)
if (affine_mode == 'affine' and not have_affine) or (resolution == 'x16'
and not have_x16):
return redirect(
url_for('slide.slide_view',
task_id=task_id,
slide_id=slide_id,
resolution=rd_resolution,
affine_mode=rd_affine_mode))
# Get additional project info
pr = ProjectRef(project)
# Form the URL templates for preloading and actual dzi access, so that in JS we
# can just do a quick substitution
url_ctx = {
'project': project,
'specimen': si['specimen_name'],
'block': si['block_name'],
'slide_name': si['slide_name'],
'slide_ext': si['slide_ext'],
'mode': affine_mode,
'resource': 'XXXXX'
}
url_tmpl_preload = url_for('dzi.dzi_preload_endpoint', **url_ctx)
url_tmpl_dzi = url_for('dzi.dzi', **url_ctx)
url_tmpl_download = url_for('dzi.dzi_download', **url_ctx)
# Build a dictionary to call
context = {
'slide_id': slide_id,
'slide_info': si,
'next_slide': next_slide,
'prev_slide': prev_slide,
'stain_list': stain_list,
'affine_mode': affine_mode,
'have_affine': have_affine,
'have_x16': have_x16,
'resolution': resolution,
'seg_mode': task['mode'],
'task_id': task_id,
'project': si['project'],
'project_name': pr.disp_name,
'block_id': si['block_id'],
'url_tmpl_preload': url_tmpl_preload,
'url_tmpl_dzi': url_tmpl_dzi,
'url_tmpl_download': url_tmpl_download,
'task': task,
'fixed_box_size': get_dltrain_fixed_box_size(task),
'user_prefs': user_prefs,
'overlays': overlays
}
# Add optional fields to context
for field in ('sample_id', 'sample_cx', 'sample_cy'):
if field in request.form:
context[field] = request.form[field]
# Render the template
return render_template('slide/slide_view.html', **context)
def refresh_slide_db(project, manifest, single_specimen=None, check_hash=True):
# Database cursor
db = get_db()
# Get the project object
pr = ProjectRef(project)
# Load the manifest file
manifest_contents = load_url(manifest)
if manifest_contents is None:
logging.error("Cannot read URL or file: %s" % (manifest, ))
sys.exit(-1)
# Read from the manifest file
for line in manifest_contents.splitlines():
# Split into words
words = line.split()
if len(words) != 2:
continue
# Check for single specimen selector
specimen = line.split()[0]
if single_specimen is not None and single_specimen != specimen:
continue
url = line.split()[1]
print('Parsing specimen "%s" with URL "%s"' % (specimen, url))
# Get the lines from the URL
specimen_manifest_contents = load_url(url, os.path.dirname(manifest))
if specimen_manifest_contents is None:
logging.warning("Cannot read URL or file: %s" % (url, ))
continue
# For each line in the URL consider it as a new slide
r = csv.reader(specimen_manifest_contents.splitlines()[1:])
for spec_line in r:
# Read the elements from the string
(slide_name, stain, block, section, slide_no, cert,
tagline) = spec_line[0:7]
# Remap slide_name to string
slide_name = str(slide_name)
# Check if the slide has already been imported into the database
slide_id = pr.get_slide_by_name(slide_name)
# Get the current set of tags
tagline = tagline.lower().strip()
tags = set(tagline.split(';')) if len(tagline) > 0 else set()
# If the slide is marked as a duplicate, we may need to delete it but regardless
# we do not proceed further
if cert == 'duplicate':
# If the slide is a duplicate, we should make it disappear
# but the problem is that we might have already done some annotation
# for that slide. I guess it still makes sense to delete the slide
if slide_id is not None:
print('DELETING slide %s as DUPLICATE' % (slide_name, ))
db.execute('DELETE FROM slide WHERE id=?', (slide_id, ))
db.commit()
# Stop processing slide
continue
# If non-duplicate slide exists, we need to check its metadata against the database
if slide_id is not None:
print('Slide %s already in the database with id=%s' %
(slide_name, slide_id))
# Check if the metadata matches
t0 = db.execute(
'SELECT * FROM slide '
'WHERE section=? AND slide=? AND stain=? AND id=?',
(section, slide_no, stain, slide_id)).fetchone()
# We may need to update the properties
if t0 is None:
print('UPDATING metadata for slide %s' % (slide_name, ))
db.execute(
'UPDATE slide SET section=?, slide=?, stain=? '
'WHERE id=?', (section, slide_no, stain, slide_id))
db.commit()
# We may also need to update the specimen/block id
t1 = db.execute(
'SELECT * FROM slide S '
' LEFT JOIN block B on S.block_id = B.id '
'WHERE B.specimen_name=? AND B.block_name=? AND S.id=?',
(specimen, block, slide_id)).fetchone()
# Update the specimen/block for this slide
if t1 is None:
print('UPDATING specimen/block for slide %s to %s/%s' %
(slide_name, specimen, block))
bid = db_get_or_create_block(project, specimen, block)
db.execute('UPDATE slide SET block_id=? WHERE id=?',
(bid, slide_id))
db.commit()
# Finally, we may need to update the tags
current_tags = set()
rc = db.execute(
'SELECT tag FROM slide_tags WHERE slide=? AND external=1',
(slide_id, ))
for row in rc.fetchall():
current_tags.add(row['tag'])
# If tags have changed, update the tags
if tags != current_tags:
print('UPDATING tags for slide %s to %s' %
(slide_name, str(tags)))
db.execute(
'DELETE FROM slide_tags WHERE slide=? AND external=1',
(slide_id, ))
for t in tags:
db.execute(
'INSERT INTO slide_tags(slide, tag, external) VALUES (?, ?, 1)',
(slide_id, t))
db.commit()
# Update the slide thumbnail, etc., optionally checking against source filesystem
update_slide_derived_data(slide_id, check_hash)
continue
# Create a slideref for this object. The way we have set all of this up,
# the extension is not coded anywhere in the manifests, so we dynamically
# check for multiple extensions
found = False
for slide_ext in ('svs', 'tif', 'tiff'):
sr = SlideRef(pr, specimen, block, slide_name, slide_ext)
if sr.resource_exists('raw', False):
# The raw slide has been found, so the slide will be entered into the database.
sid = db_create_slide(project, specimen, block, section,
slide_no, stain, slide_name,
slide_ext, tags)
print(
'Slide %s located with url %s and assigned new id %d' %
(slide_name, sr.get_resource_url('raw', False), sid))
# Update thumbnail and such
update_slide_derived_data(sid, True)
found = True
break
# We are here because no URL was found
if not found:
print('Raw image was not found for slide "%s"' %
(slide_name, ))
# Refresh slice index for all tasks in this project
rebuild_project_slice_indices(project)