def thumbnail(cls, token, ofnm, width, height, **kw):
'''converts input filename into output thumbnail'''
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return None
log.debug('Thumbnail: %s %s %s for [%s]', width, height, series, ifnm)
fmt = kw.get('fmt', 'jpeg').upper()
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return None
img = slide.get_thumbnail((width, height))
try:
img.save(ofnm, fmt)
except IOError:
tmp = '%s.tif' % ofnm
img.save(tmp, 'TIFF')
ConverterImgcnv.thumbnail(ProcessToken(ifnm=tmp),
ofnm=ofnm,
width=width,
height=height,
**kw)
slide.close()
misc.end_nounicode_win(tmp)
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# make sure the file was written
with Locks(ofnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
return ofnm
def run_read(cls, ifnm, command):
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
#command, tmp = misc.start_nounicode_win(ifnm, command)
log.debug('run_read dylib command: %s', misc.tounicode(command))
#out = cls.run_command( command )
#misc.end_nounicode_win(tmp)
retcode, out = call_imgcnvlib(command)
if retcode == 100 or retcode == 101: # some error in libbioimage, retry once
log.error('Libioimage retcode %s: retry once: %s', retcode,
command)
retcode, out = call_imgcnvlib(command)
#log.debug('Retcode: %s', retcode)
#log.debug('out: %s', out)
return out
def tile(cls, token, ofnm, level=None, x=None, y=None, sz=None, **kw):
'''extract tile from image
default interface:
Level,X,Y tile from input filename into output in TIFF format
alternative interface, not required to support and may return None in this case
scale=scale, x1=x1, y1=y1, x2=x2, y2=y2, arbitrary_size=False '''
# open slide driver does not support arbitrary size interface
if kw.get('arbitrary_size',
False) == True or level is None or sz is None:
return None
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return None
log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series,
ifnm)
level = misc.safeint(level, 0)
x = misc.safeint(x, 0)
y = misc.safeint(y, 0)
sz = misc.safeint(sz, 0)
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
dz = deepzoom.DeepZoomGenerator(slide,
tile_size=sz,
overlap=0)
img = dz.get_tile(dz.level_count - level - 1, (x, y))
img.save(ofnm, 'TIFF', compression='LZW')
slide.close()
misc.end_nounicode_win(tmp)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return None
# make sure the file was written
with Locks(ofnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
return ofnm
def action(self, token, arg):
'''arg = l,tnx,tny,tsz'''
if not token.isFile():
raise ImageServiceException(400, 'Tile: input is not an image...' )
level=0; tnx=0; tny=0; tsz=512;
vs = arg.split(',', 4)
if len(vs)>0 and vs[0].isdigit(): level = int(vs[0])
if len(vs)>1 and vs[1].isdigit(): tnx = int(vs[1])
if len(vs)>2 and vs[2].isdigit(): tny = int(vs[2])
if len(vs)>3 and vs[3].isdigit(): tsz = int(vs[3])
log.debug( 'Tile: l:%d, tnx:%d, tny:%d, tsz:%d' % (level, tnx, tny, tsz) )
# if input image is smaller than the requested tile size
dims = token.dims or {}
width = dims.get('image_num_x', 0)
height = dims.get('image_num_y', 0)
if width<=tsz and height<=tsz:
log.debug('Image is smaller than requested tile size, passing the whole image...')
return token
# construct a sliced filename
ifname = token.first_input_file()
base_name = '%s.tiles'%(token.data)
_mkdir( base_name )
ofname = os.path.join(base_name, '%s_%.3d_%.3d_%.3d' % (tsz, level, tnx, tny))
hist_name = os.path.join(base_name, '%s_histogram'%(tsz))
# if input image does not contain tile pyramid, create one and pass it along
if dims.get('image_num_resolution_levels', 0)<2 or dims.get('tile_num_x', 0)<1:
pyramid = '%s.pyramid.tif'%(token.data)
command = token.drainQueue()
if not os.path.exists(pyramid):
#command.extend(['-ohst', hist_name])
command.extend(['-options', 'compression lzw tiles %s pyramid subdirs'%default_tile_size])
log.debug('Generate tiled pyramid %s: from %s to %s with %s', token.resource_id, ifname, pyramid, command )
r = self.server.imageconvert(token, ifname, pyramid, fmt=default_format, extra=command)
if r is None:
raise ImageServiceException(500, 'Tile: could not generate pyramidal file' )
# ensure the file was created
with Locks(pyramid, failonread=(not block_tile_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
fff = (width*height) / (10000*10000)
raise ImageServiceFuture((15*fff,30*fff))
# compute the number of pyramidal levels
# sz = max(width, height)
# num_levels = math.ceil(math.log(sz, 2)) - math.ceil(math.log(min_level_size, 2)) + 1
# scales = [1/float(pow(2,i)) for i in range(0, num_levels)]
# info = {
# 'image_num_resolution_levels': num_levels,
# 'image_resolution_level_scales': ',',join([str(i) for i in scales]),
# 'tile_num_x': default_tile_size,
# 'tile_num_y': default_tile_size,
# 'converter': ConverterImgcnv.name,
# }
# load the number of pyramidal levels from the file
info2 = self.server.getImageInfo(filename=pyramid)
info = {
'image_num_resolution_levels': info2.get('image_num_resolution_levels'),
'image_resolution_level_scales': info2.get('image_resolution_level_scales'),
'tile_num_x': info2.get('tile_num_x'),
'tile_num_y': info2.get('tile_num_y'),
'converter': info2.get('converter'),
}
log.debug('Updating original input to pyramidal version %s: %s -> %s', token.resource_id, ifname, pyramid )
token.setImage(ofname, fmt=default_format, dims=info, input=pyramid)
ifname = pyramid
# compute output tile size
dims = token.dims or {}
x = tnx * tsz
y = tny * tsz
if x>=width or y>=height:
raise ImageServiceException(400, 'Tile: tile position outside of the image: %s,%s'%(tnx, tny))
# the new tile service does not change the number of z points in the image and if contains all z will perform the operation
info = {
'image_num_x': tsz if width-x >= tsz else width-x,
'image_num_y': tsz if height-y >= tsz else height-y,
#'image_num_z': 1,
#'image_num_t': 1,
}
#log.debug('Inside pyramid dims: %s', dims)
#log.debug('Inside pyramid input: %s', token.first_input_file() )
#log.debug('Inside pyramid data: %s', token.data )
# extract individual tile from pyramidal tiled image
if dims.get('image_num_resolution_levels', 0)>1 and dims.get('tile_num_x', 0)>0:
# dima: maybe better to test converter, if imgcnv then enqueue, otherwise proceed with the converter path
if dims.get('converter', '') == ConverterImgcnv.name:
c = self.server.converters[ConverterImgcnv.name]
r = c.tile(token, ofname, level, tnx, tny, tsz)
if r is not None:
if not os.path.exists(hist_name):
# write the histogram file is missing
c.writeHistogram(token, ofnm=hist_name)
# if decoder returned a list of operations for imgcnv to enqueue
if isinstance(r, list):
#r.extend([ '-ihst', hist_name])
token.histogram = hist_name
return self.server.enqueue(token, 'tile', ofname, fmt=default_format, command=r, dims=info)
# try other decoders to read tiles
ofname = '%s.tif'%ofname
if os.path.exists(ofname):
return token.setImage(ofname, fmt=default_format, dims=info, hist=hist_name, input=ofname)
else:
r = None
for n,c in self.server.converters.iteritems():
if n == ConverterImgcnv.name: continue
if callable( getattr(c, "tile", None) ):
r = c.tile(token, ofname, level, tnx, tny, tsz)
if r is not None:
if not os.path.exists(hist_name):
# write the histogram file if missing
c.writeHistogram(token, ofnm=hist_name)
return token.setImage(ofname, fmt=default_format, dims=info, hist=hist_name, input=ofname)
raise ImageServiceException(500, 'Tile could not be extracted')
def meta(cls, token, **kw):
ifnm = token.first_input_file()
if not cls.supported(token):
return {}
log.debug('Meta for: %s', ifnm)
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 10))
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return {}
rd = {
'format':
slide.properties.get(openslide.PROPERTY_NAME_VENDOR),
'image_num_series':
0,
'image_num_x':
slide.dimensions[0],
'image_num_y':
slide.dimensions[1],
'image_num_z':
1,
'image_num_t':
1,
'image_num_c':
3,
'image_num_resolution_levels':
slide.level_count,
'image_resolution_level_scales':
','.join([str(1.0 / i) for i in slide.level_downsamples]),
'image_pixel_format':
'unsigned integer',
'image_pixel_depth':
8,
'magnification':
slide.properties.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER),
'channel_0_name':
'red',
'channel_1_name':
'green',
'channel_2_name':
'blue',
'channel_color_0':
'255,0,0',
'channel_color_1':
'0,255,0',
'channel_color_2':
'0,0,255',
# new format
'channels/channel_00000/name':
'red',
'channels/channel_00000/color':
'255,0,0',
'channels/channel_00001/name':
'green',
'channels/channel_00001/color':
'0,255,0',
'channels/channel_00002/name':
'blue',
'channels/channel_00002/color':
'0,0,255',
}
if slide.properties.get(openslide.PROPERTY_NAME_MPP_X,
None) is not None:
rd.update({
'pixel_resolution_x':
slide.properties.get(openslide.PROPERTY_NAME_MPP_X, 0),
'pixel_resolution_y':
slide.properties.get(openslide.PROPERTY_NAME_MPP_Y, 0),
'pixel_resolution_unit_x':
'microns',
'pixel_resolution_unit_y':
'microns'
})
# custom - any other tags in proprietary files should go further prefixed by the custom parent
for k, v in slide.properties.iteritems():
rd['custom/%s' % k.replace('.', '/')] = v
slide.close()
# read metadata using imgcnv since openslide does not decode all of the info
meta = ConverterImgcnv.meta(
ProcessToken(ifnm=tmp or ifnm, series=token.series), **kw)
meta.update(rd)
rd = meta
misc.end_nounicode_win(tmp)
return rd
def info(cls, token, **kw):
'''returns a dict with file info'''
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return {}
log.debug('Info for: %s', ifnm)
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 10))
if not os.path.exists(ifnm):
return {}
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return {}
info2 = {
'format':
slide.properties[openslide.PROPERTY_NAME_VENDOR],
'image_num_series':
0,
'image_series_index':
0,
'image_num_x':
slide.dimensions[0],
'image_num_y':
slide.dimensions[1],
'image_num_z':
1,
'image_num_t':
1,
'image_num_c':
3,
'image_num_resolution_levels':
slide.level_count,
'image_resolution_level_scales':
','.join([str(1.0 / i) for i in slide.level_downsamples]),
'image_pixel_format':
'unsigned integer',
'image_pixel_depth':
8
}
if slide.properties.get(openslide.PROPERTY_NAME_MPP_X,
None) is not None:
info2.update({
'pixel_resolution_x':
slide.properties.get(openslide.PROPERTY_NAME_MPP_X, 0),
'pixel_resolution_y':
slide.properties.get(openslide.PROPERTY_NAME_MPP_Y, 0),
'pixel_resolution_unit_x':
'microns',
'pixel_resolution_unit_y':
'microns'
})
slide.close()
# read metadata using imgcnv since openslide does not decode all of the info
info = ConverterImgcnv.info(
ProcessToken(ifnm=tmp or ifnm, series=series), **kw)
misc.end_nounicode_win(tmp)
info.update(info2)
return info
return {}
def run(cls, ifnm, ofnm, args, **kw):
'''converts input filename into output using exact arguments as provided in args'''
if not cls.installed:
return None
failonread = kw.get('failonread') or (not block_reads)
tmp = None
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
command = [cls.CONVERTERCOMMAND]
command.extend(args)
log.debug('Run dylib command: %s', misc.tounicode(command))
proceed = True
if ofnm is not None and os.path.exists(
ofnm) and os.path.getsize(ofnm) > 16:
if kw.get('nooverwrite', False) is True:
proceed = False
log.warning(
'Run: output exists before command [%s], skipping',
misc.tounicode(ofnm))
else:
log.warning(
'Run: output exists before command [%s], overwriting',
misc.tounicode(ofnm))
if proceed is True:
#command, tmp = misc.start_nounicode_win(ifnm, command)
retcode, out = call_imgcnvlib(command)
#misc.end_nounicode_win(tmp)
if retcode == 100 or retcode == 101: # some error in libbioimage, retry once
log.error('Libioimage retcode %s: retry once: %s',
retcode, command)
retcode, out = call_imgcnvlib(command)
if retcode == 99:
# in case of a timeout
log.info('Run: timed-out for [%s]',
misc.tounicode(command))
if ofnm is not None and os.path.exists(ofnm):
os.remove(ofnm)
raise ImageServiceException(
412, 'Requested timeout reached')
if retcode != 0:
log.info('Run: returned [%s] for [%s]', retcode,
misc.tounicode(command))
return None
if ofnm is None:
return str(retcode)
# output file does not exist for some operations, like tiles
# tile command does not produce a file with this filename
# if not os.path.exists(ofnm):
# log.error ('Run: output does not exist after command [%s]', ofnm)
# return None
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# make sure the write of the output file have finished
if ofnm is not None and os.path.exists(ofnm):
with Locks(ofnm, failonread=failonread) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# safeguard for incorrectly converted files, sometimes only the tiff header can be written
# empty lock files are automatically removed before by lock code
if os.path.exists(
ofnm) and os.path.getsize(ofnm) < cls.MINIMUM_FILE_SIZE:
log.error(
'Run: output file is smaller than %s bytes, probably an error, removing [%s]',
cls.MINIMUM_FILE_SIZE, ofnm)
os.remove(ofnm)
return None
return ofnm
def action(self, token, arg):
if not token.isFile():
raise ImageServiceException(400, 'Roi: input is not an image...')
rois = []
for a in arg.split(';'):
vs = a.split(',', 4)
x1 = int(vs[0]) if len(vs) > 0 and vs[0].isdigit() else 0
y1 = int(vs[1]) if len(vs) > 1 and vs[1].isdigit() else 0
x2 = int(vs[2]) if len(vs) > 2 and vs[2].isdigit() else 0
y2 = int(vs[3]) if len(vs) > 3 and vs[3].isdigit() else 0
rois.append((x1, y1, x2, y2))
x1, y1, x2, y2 = rois[0]
if x1 <= 0 and x2 <= 0 and y1 <= 0 and y2 <= 0:
raise ImageServiceException(400, 'ROI: region is not provided')
ifile = token.first_input_file()
otemp = token.data
ofile = '%s.roi_%d,%d,%d,%d' % (token.data, x1 - 1, y1 - 1, x2 - 1,
y2 - 1)
log.debug('ROI %s: %s to %s', token.resource_id, ifile, ofile)
if len(rois) == 1:
info = {
'image_num_x': x2 - x1,
'image_num_y': y2 - y1,
}
command = [
'-roi',
'%s,%s,%s,%s' % (x1 - 1, y1 - 1, x2 - 1, y2 - 1)
]
return self.server.enqueue(token,
'roi',
ofile,
fmt=default_format,
command=command,
dims=info)
# remove pre-computed ROIs
rois = [
(_x1, _y1, _x2, _y2) for _x1, _y1, _x2, _y2 in rois
if not os.path.exists('%s.roi_%d,%d,%d,%d' %
(otemp, _x1 - 1, _y1 - 1, _x2 - 1, _y2 - 1))
]
lfile = '%s.rois' % (otemp)
command = token.drainQueue()
if not os.path.exists(ofile) or len(rois) > 0:
# global ROI lock on this input since we can't lock on all individual outputs
with Locks(ifile, lfile, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
s = ';'.join([
'%s,%s,%s,%s' % (x1 - 1, y1 - 1, x2 - 1, y2 - 1)
for x1, y1, x2, y2 in rois
])
command.extend(['-roi', s])
command.extend(
['-template',
'%s.roi_{x1},{y1},{x2},{y2}' % otemp])
self.server.imageconvert(token,
ifile,
ofile,
fmt=default_format,
extra=command)
# ensure the virtual locking file is not removed
with open(lfile, 'wb') as f:
f.write('#Temporary locking file')
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# ensure the operation is finished
if os.path.exists(lfile):
with Locks(lfile, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
info = {
'image_num_x': x2 - x1,
'image_num_y': y2 - y1,
}
return token.setImage(ofile,
fmt=default_format,
dims=info,
input=ofile)