def dryrun(self, token, arg):
ss = arg.split(',')
size = [safeint(ss[0], 128) if len(ss)>0 else 128,
safeint(ss[1], 128) if len(ss)>1 else 128]
method = ss[2].upper() if len(ss)>2 and len(ss[2])>0 else 'BC'
preproc = ss[3].lower() if len(ss)>3 and len(ss[3])>0 else ''
preprocc = ',%s'%preproc if len(preproc)>0 else '' # attempt to keep the filename backward compatible
fmt = ss[4].lower() if len(ss)>4 and len(ss[4])>0 else 'jpeg'
dims = token.dims or {}
num_x = int(dims.get('image_num_x', 0))
num_y = int(dims.get('image_num_y', 0))
try:
width, height = compute_new_size(num_x, num_y, size[0], size[1], keep_aspect_ratio=True, no_upsample=True)
except ZeroDivisionError:
log.warning('Thumbnail warning while guessing size %s: [%sx%s] to [%sx%s]', token.resource_id, num_x, num_y, width, height)
width = size[0]
height = size[1]
info = {
'image_num_x': width,
'image_num_y': height,
'image_num_c': 3,
'image_num_z': 1,
'image_num_t': 1,
'image_pixel_depth': 8,
'format': fmt,
}
ext = self.server.converters.defaultExtension(fmt)
ofile = '%s.thumb_%s,%s,%s%s.%s'%(token.data, size[0], size[1], method, preprocc, ext)
#log.debug('Dryrun thumbnail [%s]', ofile)
return token.setImage(ofile, fmt=fmt, dims=info, input=ofile)
def action(self, token, arg):
method = 'a'
num_channels = 3
arg = arg.lower()
if arg == 'display':
args = ['-fusemeta']
argenc = 'display'
elif arg == 'gray' or arg == 'grey':
args = ['-fusegrey']
num_channels = 1
argenc = 'gray'
else:
args = ['-fusergb', arg]
if ':' in arg:
(arg, method) = arg.split(':', 1)
elif '.' in arg:
(arg, method) = arg.split('.', 1)
argenc = ''.join([
hex(int(i)).replace('0x', '')
for i in arg.replace(';', ',').split(',') if i is not ''
])
# test if all channels are valid
dims = token.dims or {}
img_num_c = misc.safeint(dims.get('image_num_c'), 0)
groups = [i for i in arg.split(';') if i is not '']
# if there are more groups than input channels
if len(groups) > img_num_c:
groups = groups[img_num_c:]
for i, g in enumerate(groups):
channels = [misc.safeint(i, 0) for i in g.split(',')]
# we can skip 0 weighted channels even if they are outside of the image channel range
if max(channels) > 0:
raise ImageServiceException(
400,
'Fuse: requested fusion of channel outside bounds %s>%s (%s)'
% (img_num_c + i, img_num_c, g))
if method != 'a':
args.extend(['-fusemethod', method])
info = {
'image_num_c': num_channels,
}
ifile = token.first_input_file()
ofile = '%s.fuse_%s_%s' % (token.data, argenc, method)
log.debug('Fuse %s: %s to %s with [%s:%s]', token.resource_id, ifile,
ofile, arg, method)
return self.server.enqueue(token,
'fuse',
ofile,
fmt=default_format,
command=args,
dims=info)
def tile(cls, token, ofnm, level, x, y, sz, **kw):
'''extract tile Level,X,Y tile from input filename into output in OME-TIFF format'''
# imgcnv driver does not support arbitrary size interface
if kw.get('arbitrary_size',
False) == True or level is None or sz is None:
return None
ifnm = token.first_input_file()
series = token.series
page = 0
log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series,
ifnm)
info = token.dims or {}
tile_w = info.get('tile_num_x', 0)
tile_h = info.get('tile_num_y', 0)
num_l = info.get('image_num_resolution_levels', 1)
if num_l <= 1 or tile_w < 1 or tile_h < 1:
log.debug('Image does not contain tiles, skipping...')
return None
queue = token.getQueue()
#command = ['-o', ofnm, '-t', 'tiff']
command = []
if token.series is not None and token.series != 0:
command.extend(['-path', token.series])
# separate normal and multi-file series
if '-i' not in queue and '-il' not in queue and '-page' not in queue:
if token.is_multifile_series() is False:
command.extend(['-i', ifnm])
command.extend(['-page', str(page + 1)])
else:
# use first image of the series, need to check for separate channels here
files = token.input
meta = token.meta or {}
samples = meta.get('image_num_c', 0)
if samples < 2:
command.extend(['-i', files[page]])
else:
# in case of channels being stored in separate files
page = page * samples
command.extend(['-i', files[page]])
for s in range(1, samples):
command.extend(['-c', files[page + s]])
level = misc.safeint(level, 0)
x = misc.safeint(x, 0)
y = misc.safeint(y, 0)
sz = misc.safeint(sz, 0)
command.extend(['-tile', '%s,%s,%s,%s' % (sz, x, y, level)])
# add speed file
command.extend(['-speed', token.get_speed_file()])
return command
def get_formats(cls):
'''inits supported file formats'''
if cls.installed_formats is None:
formats_xml = cls.run_command([cls.CONVERTERCOMMAND, '-fmtxml'])
formats = etree.fromstring('<formats>%s</formats>' % formats_xml)
cls.installed_formats = OrderedDict()
codecs = formats.xpath('//codec')
for c in codecs:
try:
name = c.get('name')
fullname = c.xpath('tag[@name="fullname"]')[0].get(
'value', '')
exts = c.xpath('tag[@name="extensions"]')[0].get(
'value', '').split('|')
reading = len(
c.xpath(
'tag[@name="support" and @value="reading"]')) > 0
writing = len(
c.xpath(
'tag[@name="support" and @value="writing"]')) > 0
multipage = len(
c.xpath(
'tag[@name="support" and @value="writing multiple pages"]'
)) > 0
metadata = len(
c.xpath(
'tag[@name="support" and @value="reading metadata"]'
)
) > 0 or len(
c.xpath(
'tag[@name="support" and @value="writing metadata"]'
)) > 0
samples_min = misc.safeint(
c.xpath('tag[@name="min-samples-per-pixel"]')[0].get(
'value', '0'))
samples_max = misc.safeint(
c.xpath('tag[@name="max-samples-per-pixel"]')[0].get(
'value', '0'))
bits_min = misc.safeint(
c.xpath('tag[@name="min-bits-per-sample"]')[0].get(
'value', '0'))
bits_max = misc.safeint(
c.xpath('tag[@name="max-bits-per-sample"]')[0].get(
'value', '0'))
except IndexError:
continue
cls.installed_formats[name.lower()] = Format(
name=name,
fullname=fullname,
ext=exts,
reading=reading,
writing=writing,
multipage=multipage,
metadata=metadata,
samples=(samples_min, samples_max),
bits=(bits_min, bits_max))
def tile(cls, token, ofnm, level=None, x=None, y=None, sz=None, **kw):
'''extract tile from image
default interface:
Level,X,Y tile from input filename into output in TIFF format
alternative interface, not required to support and may return None in this case
scale=scale, x1=x1, y1=y1, x2=x2, y2=y2, arbitrary_size=False '''
# open slide driver does not support arbitrary size interface
if kw.get('arbitrary_size',
False) == True or level is None or sz is None:
return None
ifnm = token.first_input_file()
series = token.series
if not cls.supported(token):
return None
log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series,
ifnm)
level = misc.safeint(level, 0)
x = misc.safeint(x, 0)
y = misc.safeint(y, 0)
sz = misc.safeint(sz, 0)
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
try:
_, tmp = misc.start_nounicode_win(ifnm, [])
slide = openslide.OpenSlide(tmp or ifnm)
dz = deepzoom.DeepZoomGenerator(slide,
tile_size=sz,
overlap=0)
img = dz.get_tile(dz.level_count - level - 1, (x, y))
img.save(ofnm, 'TIFF', compression='LZW')
slide.close()
misc.end_nounicode_win(tmp)
except (openslide.OpenSlideUnsupportedFormatError,
openslide.OpenSlideError):
misc.end_nounicode_win(tmp)
return None
# make sure the file was written
with Locks(ofnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
return ofnm
def init_classes_dataset(self):
if self.training_set is None:
raise ConnoisseurException(responses.BAD_REQUEST, 'Cannot initialize classes due to missing training dataset')
with Locks(None, self.lockable, failonexist=True) as l:
if l.locked is False: # the file is being written by another process
raise ConnoisseurException(responses.LOCKED, 'The model is locked for processing by another process')
try:
dataset_url = ensure_url(self.training_set)
adapter_gobs = self.create_adapter_gobs(model=self, image=None)
classes = {}
gobs = data_service.query(resource_type='value', parent=dataset_url, extract='gobject[type]')
idx = 0
self.total_samples = 0
for g in gobs:
k = g.get('type')
n = misc.safeint(g.text, 0)
if k is None: continue
k = adapter_gobs.get_class_name(g) # adapt the class name, might need some change since the node is not a true gobject
if k is None: continue
if k not in classes:
classes[k] = {
'label': k,
'id': idx,
'samples': n,
}
idx += 1
else:
classes[k]['samples'] += n
self.total_samples += n
self.classes_data = classes
self.classes_data_by_original_id = dict((v['id'],v) for k,v in self.classes_data.iteritems())
#log.debug('Classes data: %s', str(self.classes_data))
self.classes_model = {}
self.classes_model_by_id = {}
self.classes_model_by_original_id = {}
self.number_classes_in_model = 0
except:
self.update_with_error('status.classes.init', 'Exception during init_classes_dataset')
raise
# update model resource
set_tag(self.resource, 'total_samples', self.total_samples)
set_classes(self.resource, 'classes_data', self.classes_data)
set_classes(self.resource, 'classes_model', self.classes_model)
set_tag(self.resource, 'status.classes.init', 'finished')
self.sync_resource()
def action(self, token, arg):
if not token.isFile():
raise ImageServiceException(
400, 'Pixelcounter: input is not an image...')
arg = safeint(arg.lower(), 256) - 1
ifile = token.first_input_file()
ofile = '%s.pixelcounter_%s.xml' % (token.data, arg)
log.debug('Pixelcounter %s: %s to %s with [%s]', token.resource_id,
ifile, ofile, arg)
command = token.drainQueue()
if not os.path.exists(ofile):
command.extend(['-pixelcounts', str(arg)])
self.server.imageconvert(token, ifile, ofile, extra=command)
return token.setXmlFile(fname=ofile)
def dryrun(self, token, arg):
# parse arguments
#log.debug('Dryrun Slice')
x1 = 0
x2 = 0
y1 = 0
y2 = 0
z1 = 0
z2 = 0
t1 = 0
t2 = 0
extended_dims = {}
extended_str = ''
if ':' not in arg:
# the standard way of defining dimensions
vs = [[safeint(i, 0) for i in vs.split('-', 1)]
for vs in arg.split(',')]
for v in vs:
if len(v) < 2: v.append(0)
for v in range(len(vs) - 4):
vs.append([0, 0])
x1, x2 = vs[0]
y1, y2 = vs[1]
z1, z2 = vs[2]
t1, t2 = vs[3]
else:
# the extended way of defining dimensions
for a in arg.split(','):
k, v = a.split(':', 1)
if k not in self.dimension_names: continue
extended_dims[k] = [safeint(i, 0) for i in v.split('-', 1)]
# update x,y,z and t from the new style args
x1, x2 = pop_dimension_range(extended_dims, 'x')
y1, y2 = pop_dimension_range(extended_dims, 'y')
z1, z2 = pop_dimension_range(extended_dims, 'z')
t1, t2 = pop_dimension_range(extended_dims, 't')
# update extended string part
if len(extended_dims) > 0:
# get a string of extended dimensions in the defined order
ddd = [
get_dimension_string(extended_dims, k)
for k in self.dimension_names if k in extended_dims
]
extended_str = '.%s' % (','.join(ddd))
# in case slices request an exact copy, skip
if x1 == 0 and x2 == 0 and y1 == 0 and y2 == 0 and z1 == 0 and z2 == 0 and t1 == 0 and t2 == 0 and len(
extended_dims) < 1:
#log.debug('Dryrun Slice: skipping due to pars==0')
return token
dims = token.dims or {}
if x1 <= 1 and x2 >= dims.get('image_num_x', 1):
x1 = 0
x2 = 0
if y1 <= 1 and y2 >= dims.get('image_num_y', 1):
y1 = 0
y2 = 0
if z1 <= 1 and z2 >= dims.get('image_num_z', 1):
z1 = 0
z2 = 0
if t1 <= 1 and t2 >= dims.get('image_num_t', 1):
t1 = 0
t2 = 0
# check image bounds - dima: comment since dryrun may not contain proper dimensions
# if z1>dims.get('image_num_z', 1):
# raise ImageServiceException(400, 'Slice: requested Z slice outside of image bounds: [%s]'%z1 )
# if z2>dims.get('image_num_z', 1):
# raise ImageServiceException(400, 'Slice: requested Z slice outside of image bounds: [%s]'%z2 )
# if t1>dims.get('image_num_t', 1):
# raise ImageServiceException(400, 'Slice: requested T plane outside of image bounds: [%s]'%t1 )
# if t2>dims.get('image_num_t', 1):
# raise ImageServiceException(400, 'Slice: requested T plane outside of image bounds: [%s]'%t2 )
# shortcuts are only possible with no ROIs are requested
if x1 == x2 == 0 and y1 == y2 == 0 and len(dims) > 0:
# shortcut if input image has only one T and Z
if dims.get('image_num_z', 1) <= 1 and dims.get(
'image_num_t', 1) <= 1 and len(extended_dims) < 1:
#log.debug('Dryrun Slice: plane requested on image with no T or Z planes, skipping...')
return token
# shortcut if asking for all slices with only a specific time point in an image with only one time pont
if z1 == z2 == 0 and t1 <= 1 and dims.get(
'image_num_t', 1) <= 1 and len(extended_dims) < 1:
#log.debug('Dryrun Slice: T plane requested on image with no T planes, skipping...')
return token
# shortcut if asking for all time points with only a specific z slice in an image with only one z slice
if t1 == t2 == 0 and z1 <= 1 and dims.get(
'image_num_z', 1) <= 1 and len(extended_dims) < 1:
#log.debug('Dryrun Slice: Z plane requested on image with no Z planes, skipping...')
return token
if z1 == z2 == 0:
z1 = 1
z2 = dims.get('image_num_z', 1)
if t1 == t2 == 0:
t1 = 1
t2 = dims.get('image_num_t', 1)
new_w = x2 - x1
new_h = y2 - y1
new_z = max(1, z2 - z1 + 1)
new_t = max(1, t2 - t1 + 1)
info = {
'image_num_z': new_z,
'image_num_t': new_t,
}
if new_w > 0: info['image_num_x'] = new_w + 1
if new_h > 0: info['image_num_y'] = new_h + 1
ofname = '%s.%d-%d,%d-%d,%d-%d,%d-%d%s.ome.tif' % (
token.data, x1, x2, y1, y2, z1, z2, t1, t2, extended_str)
#log.debug('Dryrun Slice [%s]', ofname)
return token.setImage(ofname,
fmt=default_format,
dims=info,
input=ofname)
def action(self, token, arg):
'''arg = x1-x2,y1-y2,z|z1-z2,t|t1-t2
or
arg = z:V1-V2,fov:V,...
'''
if not token.isFile():
raise ImageServiceException(400, 'Slice: input is not an image...')
# parse arguments
x1 = 0
x2 = 0
y1 = 0
y2 = 0
z1 = 0
z2 = 0
t1 = 0
t2 = 0
extended_dims = {}
extended_str = ''
if ':' not in arg:
# the standard way of defining dimensions
vs = [[safeint(i, 0) for i in vs.split('-', 1)]
for vs in arg.split(',')]
for v in vs:
if len(v) < 2: v.append(0)
for v in range(len(vs) - 4):
vs.append([0, 0])
x1, x2 = vs[0]
y1, y2 = vs[1]
z1, z2 = vs[2]
t1, t2 = vs[3]
else:
# the extended way of defining dimensions
for a in arg.split(','):
k, v = a.split(':', 1)
if k not in self.dimension_names: continue
extended_dims[k] = [safeint(i, 0) for i in v.split('-', 1)]
# update x,y,z and t from the new style args
x1, x2 = pop_dimension_range(extended_dims, 'x')
y1, y2 = pop_dimension_range(extended_dims, 'y')
z1, z2 = pop_dimension_range(extended_dims, 'z')
t1, t2 = pop_dimension_range(extended_dims, 't')
# update extended string part
if len(extended_dims) > 0:
# get a string of extended dimensions in the defined order
ddd = [
get_dimension_string(extended_dims, k)
for k in self.dimension_names if k in extended_dims
]
extended_str = '.%s' % (','.join(ddd))
# in case slices request an exact copy, skip
if x1 == 0 and x2 == 0 and y1 == 0 and y2 == 0 and z1 == 0 and z2 == 0 and t1 == 0 and t2 == 0 and len(
extended_dims) < 1:
return token
dims = token.dims or {}
if x1 <= 1 and x2 >= dims.get('image_num_x', 1):
x1 = 0
x2 = 0
if y1 <= 1 and y2 >= dims.get('image_num_y', 1):
y1 = 0
y2 = 0
if z1 <= 1 and z2 >= dims.get('image_num_z', 1):
z1 = 0
z2 = 0
if t1 <= 1 and t2 >= dims.get('image_num_t', 1):
t1 = 0
t2 = 0
# check image bounds
if z1 > dims.get('image_num_z', 1):
raise ImageServiceException(
400,
'Slice: requested Z slice outside of image bounds: [%s]' % z1)
if z2 > dims.get('image_num_z', 1):
raise ImageServiceException(
400,
'Slice: requested Z slice outside of image bounds: [%s]' % z2)
if t1 > dims.get('image_num_t', 1):
raise ImageServiceException(
400,
'Slice: requested T plane outside of image bounds: [%s]' % t1)
if t2 > dims.get('image_num_t', 1):
raise ImageServiceException(
400,
'Slice: requested T plane outside of image bounds: [%s]' % t2)
# shortcuts are only possible with no ROIs are requested
if x1 == x2 == 0 and y1 == y2 == 0:
# shortcut if input image has only one T and Z
if dims.get('image_num_z', 1) <= 1 and dims.get(
'image_num_t', 1) <= 1 and len(extended_dims) < 1:
log.debug(
'Slice: plane requested on image with no T or Z planes, skipping...'
)
return token
# shortcut if asking for all slices with only a specific time point in an image with only one time pont
if z1 == z2 == 0 and t1 <= 1 and dims.get(
'image_num_t', 1) <= 1 and len(extended_dims) < 1:
log.debug(
'Slice: T plane requested on image with no T planes, skipping...'
)
return token
# shortcut if asking for all time points with only a specific z slice in an image with only one z slice
if t1 == t2 == 0 and z1 <= 1 and dims.get(
'image_num_z', 1) <= 1 and len(extended_dims) < 1:
log.debug(
'Slice: Z plane requested on image with no Z planes, skipping...'
)
return token
if z1 == z2 == 0:
z1 = 1
z2 = dims.get('image_num_z', 1)
if t1 == t2 == 0:
t1 = 1
t2 = dims.get('image_num_t', 1)
# construct a sliced filename
ifname = token.first_input_file()
ofname = '%s.%d-%d,%d-%d,%d-%d,%d-%d%s.ome.tif' % (
token.data, x1, x2, y1, y2, z1, z2, t1, t2, extended_str)
log.debug('Slice %s: from [%s] to [%s]', token.resource_id, ifname,
ofname)
new_w = x2 - x1
new_h = y2 - y1
new_z = max(1, z2 - z1 + 1)
new_t = max(1, t2 - t1 + 1)
info = {
'image_num_z': new_z,
'image_num_t': new_t,
}
if new_w > 0: info['image_num_x'] = new_w + 1
if new_h > 0: info['image_num_y'] = new_h + 1
meta = token.meta or {}
unsupported_multifile = False
if token.is_multifile_series() is True and (z2 == 0 or z2 == z1) and (
t2 == 0 or t2 == t1) and x1 == x2 and y1 == y2 and meta.get(
'image_num_c', 0) == 0:
unsupported_multifile = True
if dims.get(
'converter',
'') == ConverterImgcnv.name or unsupported_multifile is True:
r = ConverterImgcnv.slice(token,
ofname,
z=(z1, z2),
t=(t1, t2),
roi=(x1, x2, y1, y2),
fmt=default_format,
**extended_dims)
# if decoder returned a list of operations for imgcnv to enqueue
if isinstance(r, list):
return self.server.enqueue(token,
'slice',
ofname,
fmt=default_format,
command=r,
dims=info)
# slice the image
if not os.path.exists(ofname):
intermediate = '%s.ome.tif' % token.data
if 'converter' in dims and dims.get(
'converter') in self.server.converters:
r = self.server.converters[dims.get('converter')].slice(
token,
ofname,
z=(z1, z2),
t=(t1, t2),
roi=(x1, x2, y1, y2),
fmt=default_format,
intermediate=intermediate)
# if desired converter failed, perform exhaustive conversion
if r is None:
for n, c in self.server.converters.iteritems():
if n in [ConverterImgcnv.name, dims.get('converter')]:
continue
r = c.slice(token,
ofname,
z=(z1, z2),
t=(t1, t2),
roi=(x1, x2, y1, y2),
fmt=default_format,
intermediate=intermediate)
if r is not None:
break
if r is None:
log.error('Slice %s: could not generate slice for [%s]',
token.resource_id, ifname)
raise ImageServiceException(415, 'Could not generate slice')
# if decoder returned a list of operations for imgcnv to enqueue
if isinstance(r, list):
return self.server.enqueue(token,
'slice',
ofname,
fmt=default_format,
command=r,
dims=info)
return token.setImage(ofname,
fmt=default_format,
dims=info,
input=ofname)
def meta(cls, token, **kw):
if not cls.installed:
return {}
ifnm = token.first_input_file()
series = int(token.series)
log.debug('Meta for: %s', ifnm )
if os.name == 'nt':
nulldevice = 'NUL'
elif os.name == 'posix':
nulldevice = '/dev/null'
command = [cls.CONVERTERCOMMAND, '-i', ifnm, '-m', '-l', nulldevice, '-ii', '%s'%series]
command.extend( cls.extension(token, **kw) ) # extend if timeout or meta are present
meta = cls.run_read(ifnm, command)
if meta is None:
return {}
# fix a bug in Imaris Convert exporting XML with invalid chars
# by removing the <ImplParameters> tag
# params is formatted in INI format
params = ''
try:
p = misc.between(BLOCK_START, BLOCK_END, meta)
meta = meta.replace('%s%s%s'%(BLOCK_START, p, BLOCK_END), '', 1)
params = p
except UnboundLocalError:
return {}
########################################
# Parse Meta XML
# most files have improper encodings, try to recover
########################################
rd = {}
try:
mee = etree.fromstring(meta)
except etree.XMLSyntaxError:
try:
mee = etree.fromstring(meta, parser=etree.XMLParser(encoding='iso-8859-1'))
except (etree.XMLSyntaxError, LookupError):
try:
mee = etree.fromstring(meta, parser=etree.XMLParser(encoding='utf-16'))
except (etree.XMLSyntaxError, LookupError):
try:
mee = etree.fromstring(meta, parser=etree.XMLParser(recover=True))
except etree.XMLSyntaxError:
log.error ("Unparsable %s", meta)
return {}
if '<FileInfo2>' in meta: # v7
rd['image_num_series'] = misc.safeint(misc.xpathtextnode(mee, '/FileInfo2/NumberOfImages'), 1)
imagenodepath = '/FileInfo2/Image[@mIndex="%s"]'%series
else: # v8
rd['image_num_series'] = misc.safeint(misc.xpathtextnode(mee, '/MetaData/NumberOfImages'), 1)
imagenodepath = '/MetaData/Image[@mIndex="%s"]'%series
rd['image_series_index'] = series
rd['date_time'] = misc.xpathtextnode(mee, '%s/ImplTimeInfo'%imagenodepath).split(';', 1)[0]
#rd['format'] = misc.xpathtextnode(mee, '%s/BaseDescription'%imagenodepath).split(':', 1)[1].strip(' ')
rd['format'] = misc.xpathtextnode(mee, '%s/BaseDescription'%imagenodepath) #.split(':', 1)[1].strip(' ')
# dims
dims = misc.xpathtextnode(mee, '%s/BaseDimension'%imagenodepath).split(' ')
try:
rd['image_num_x'] = misc.safeint(dims[0])
rd['image_num_y'] = misc.safeint(dims[1])
rd['image_num_z'] = misc.safeint(dims[2])
rd['image_num_c'] = misc.safeint(dims[3])
rd['image_num_t'] = misc.safeint(dims[4])
except IndexError:
pass
# pixel format
pixeltypes = {
'uint8': ('unsigned integer', 8),
'uint16': ('unsigned integer', 16),
'uint32': ('unsigned integer', 32),
'uint64': ('unsigned integer', 64),
'int8': ('signed integer', 8),
'int16': ('signed integer', 16),
'int32': ('signed integer', 32),
'int64': ('signed integer', 64),
'float': ('floating point', 32),
'double': ('floating point', 64),
}
try:
t = pixeltypes[misc.xpathtextnode(mee, '%s/ImplDataType'%imagenodepath).lower()]
rd['image_pixel_format'] = t[0]
rd['image_pixel_depth'] = t[1]
except KeyError:
pass
# resolution
extmin = [misc.safefloat(i) for i in misc.xpathtextnode(mee, '%s/ImplExtendMin'%imagenodepath).split(' ')]
extmax = [misc.safefloat(i) for i in misc.xpathtextnode(mee, '%s/ImplExtendMax'%imagenodepath).split(' ')]
rd['pixel_resolution_x'] = (extmax[0]-extmin[0])/rd['image_num_x']
rd['pixel_resolution_y'] = (extmax[1]-extmin[1])/rd['image_num_y']
rd['pixel_resolution_z'] = (extmax[2]-extmin[2])/rd['image_num_z']
# Time resolution is apparently missing in Imaris XML
#rd['pixel_resolution_z'] = (extmax[2]-extmin[2])/rd['image_num_z']
rd['pixel_resolution_unit_x'] = 'microns'
rd['pixel_resolution_unit_y'] = 'microns'
rd['pixel_resolution_unit_z'] = 'microns'
########################################
# Parse params INI
########################################
#params = misc.xpathtextnode(mee, '%s/ImplParameters'%imagenodepath)
# use index 0 since we fetch meta data with imageindex argument
sp = StringIO.StringIO(params)
config = ConfigParser.ConfigParser()
try:
config.readfp(sp)
except Exception:
safe_config_read(config, sp)
sp.close()
# custom - any tags in proprietary files should go further prefixed by the custom parent
for section in config.sections():
for option in config.options(section):
rd['custom/%s/%s'%(section,option)] = config.get(section, option)
# Image parameters
safeReadAndSet(config, 'Image', 'numericalaperture', rd, 'numerical_aperture')
# channel names, colors and other info
for c in range(rd['image_num_c']):
section = 'Channel %s'%c
path = 'channels/channel_%.5d'%c
name = safeRead(config, section, 'Name') or ''
dye = safeRead(config, section, 'Dye name') or safeRead(config, section, 'Fluor')
if dye is not None:
name = '%s (%s)'%(name, dye)
if len(name)>0:
rd['channel_%s_name'%c] = name
rgb = safeRead(config, section, 'Color')
if rgb is not None:
rd['channel_color_%s'%c] = ','.join([str(int(misc.safefloat(i)*255)) for i in rgb.split(' ')])
# new channel format
if len(name)>0:
rd['%s/name'%path] = name
if rgb is not None:
rd['%s/color'%path] = ','.join(rgb.split(' '))
safeReadAndSet(config, section, 'ColorOpacity', rd, '%s/opacity'%path)
safeReadAndSet(config, section, 'Fluor', rd, '%s/fluor'%path)
safeReadAndSet(config, section, 'GammaCorrection', rd, '%s/gamma'%path)
safeReadAndSet(config, section, 'LSMEmissionWavelength', rd, '%s/lsm_emission_wavelength'%path)
safeReadAndSet(config, section, 'LSMExcitationWavelength', rd, '%s/lsm_excitation_wavelength'%path)
safeReadAndSet(config, section, 'LSMPinhole', rd, '%s/lsm_pinhole_radius'%path)
safeReadAndSet(config, section, 'objective', rd, '%s/objective'%path)
rng = safeRead(config, section, 'ColorRange')
if rng is not None:
rd['%s/range'%path] = ','.join(rng.split(' '))
# read Zeiss CZI specific metadata not parsed properly by the Imaris convert
qpath = 'custom/ZeissAttrs/imagedocument/metadata/information/image/dimensions/channels/channel'
# color as defined in CZI and not properly red by ImarisConvert
t = '%s/color %s'%(qpath, c)
if t in rd:
#<tag name="color 0" value="#FF0000FF"/>
try:
v = rd[t]
r,g,b,a = imaris_to_rgb(v)
#new
rd['%s/color'%path] = '%s,%s,%s'%(r/255.0, g/255.0, b/255.0)
# old
rd['channel_color_%s'%c] = '%s,%s,%s'%(r, g, b)
except Exception:
pass
# channel exposure defined in CZI
t = '%s/exposuretime %s'%(qpath, c)
if t in rd:
try:
#<tag name="exposuretime 0" value="53000000"/> -> 53.0 ms
v = misc.safefloat(rd[t])
rd['%s/exposure'%path] = v/1000000.0
rd['%s/exposure_units'%path] = 'ms'
except Exception:
pass
# instrument and assay names, not standard CZI but used in industry
path = 'custom/Document'
qpath = 'custom/ZeissAttrs/imagedocument/metadata/experiment/experimentblocks/acquisitionblock/processinggraph/filters/filter'
t = '%s/argument'%(qpath)
if t in rd:
#"INSTRUMENT_NAME:Axio2;ASSAY_NAME:ARv7;FILTER_NAME:Epic.AnalysisFilters.ARv7.dll"
v = rd[t]
try:
v = [i.split(':') for i in v.split(';')]
for i in v:
rd['%s/%s'%(path, i[0].lower())] = i[1]
except Exception:
pass
# slide id found in barcode
path = 'custom/Document'
qpath = 'custom/ZeissAttrs/imagedocument/metadata/attachmentinfos/attachmentinfo/label/barcodes/barcode'
t = '%s/content'%(qpath)
if t in rd:
rd['%s/slide_id'%path] = rd[t]
return rd
def meta(cls, token, **kw):
if not cls.installed:
return {}
ifnm = token.first_input_file()
series = token.series
if not os.path.exists(ifnm):
return {}
log.debug('Meta for: %s', ifnm )
o = cls.run_read(ifnm, [cls.BFINFO, '-nopix', '-omexml', '-novalid', '-no-upgrade', '-series', '%s'%series, ifnm] )
if o is None:
return {}
# extract the OME-XML part
try:
omexml = misc.between('<OME', '</OME>', o)
omexml = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n<OME%s</OME>'%omexml
except UnboundLocalError:
return {}
########################################
# Parse non XML parts
########################################
rd = {}
rd['image_num_series'] = misc.safeint(misc.between('Series count =', '\n', o),1)
rd['image_series_index'] = series
rd['format'] = misc.between('Checking file format [', ']', o)
########################################
# Parse Meta XML
# bioformats defines UTF-8 encoding, but it may be something else, try to recover
########################################
try:
mee = etree.fromstring(omexml)
except etree.XMLSyntaxError:
try:
mee = etree.fromstring(omexml, parser=etree.XMLParser(encoding='iso-8859-1'))
except (etree.XMLSyntaxError, LookupError):
try:
mee = etree.fromstring(omexml, parser=etree.XMLParser(encoding='cp1252'))
except (etree.XMLSyntaxError, LookupError):
mee = etree.fromstring(omexml, parser=etree.XMLParser(recover=True))
imagenodepath = 'ome:Image[@ID="Image:%s"]'%(series)
namespaces = {
'ome': misc.between('OME xmlns="', '"', omexml),
'sa': misc.between('StructuredAnnotations xmlns="', '"', omexml),
#'om': misc.between('OriginalMetadata xmlns="', '"', o), # dima: v4.x.x
}
rd['date_time'] = misc.xpathtextnode(mee, '%s/ome:AcquisitionDate'%imagenodepath, namespaces=namespaces).replace('T', ' ')
pixels = mee.xpath('%s/ome:Pixels'%imagenodepath, namespaces=namespaces)[0]
rd['image_num_x'] = misc.safeint(pixels.get('SizeX', '0').split(' ')[0])
rd['image_num_y'] = misc.safeint(pixels.get('SizeY', '0').split(' ')[0])
rd['image_num_z'] = misc.safeint(pixels.get('SizeZ', '0').split(' ')[0])
rd['image_num_c'] = misc.safeint(pixels.get('SizeC', '0').split(' ')[0]) # fix for '3 (effectively 1)' as number of channels
rd['image_num_t'] = misc.safeint(pixels.get('SizeT', '0').split(' ')[0])
# pixel format
pixeltypes = {
'uint8': ('unsigned integer', 8),
'uint16': ('unsigned integer', 16),
'uint32': ('unsigned integer', 32),
'uint64': ('unsigned integer', 64),
'int8': ('signed integer', 8),
'int16': ('signed integer', 16),
'int32': ('signed integer', 32),
'int64': ('signed integer', 64),
'float': ('floating point', 32),
'double': ('floating point', 64),
}
try:
t = pixeltypes[pixels.get('Type', 0).lower()]
rd['image_pixel_format'] = t[0]
rd['image_pixel_depth'] = t[1]
except KeyError:
pass
# resolution
rd['pixel_resolution_x'] = misc.safefloat(pixels.get('PhysicalSizeX', '0.0'))
rd['pixel_resolution_y'] = misc.safefloat(pixels.get('PhysicalSizeY', '0.0'))
rd['pixel_resolution_z'] = misc.safefloat(pixels.get('PhysicalSizeZ', '0.0'))
rd['pixel_resolution_t'] = misc.safefloat(pixels.get('TimeIncrement', '0.0'))
rd['pixel_resolution_unit_x'] = 'microns'
rd['pixel_resolution_unit_y'] = 'microns'
rd['pixel_resolution_unit_z'] = 'microns'
rd['pixel_resolution_unit_t'] = 'seconds'
# default channel mapping
if rd.get('image_num_c', 0) == 1:
rd.setdefault('channel_color_0', '255,255,255')
# channel info
channels = mee.xpath('%s/ome:Pixels/ome:Channel'%imagenodepath, namespaces=namespaces)
for c,i in zip(channels, range(len(channels))):
bfReadAndSet(c, 'Name', rd, 'channel_%s_name'%i, defval='ch_%s'%i)
bfReadAndSet(c, 'Color', rd, 'channel_color_%s'%i, f=bfColorToString)
# new format
for c,i in zip(channels, range(len(channels))):
path = 'channels/channel_%.5d'%i
bfReadAndSet(c, 'Name', rd, '%s/name'%path, defval='ch_%s'%i)
bfReadAndSet(c, 'Color', rd, '%s/color'%path, f=bfColorToString)
bfReadAndSet(c, 'Fluor', rd, '%s/fluor'%path)
bfReadAndSet(c, 'EmissionWavelength', rd, '%s/lsm_emission_wavelength'%path)
bfReadAndSet(c, 'ExcitationWavelength', rd, '%s/lsm_excitation_wavelength'%path)
bfReadAndSet(c, 'ContrastMethod', rd, '%s/contrast_method'%path)
bfReadAndSet(c, 'AcquisitionMode', rd, '%s/acquisition_mode'%path)
bfReadAndSet(c, 'IlluminationType', rd, '%s/illumination'%path)
bfReadAndSet(c, 'PinholeSize', rd, '%s/pinhole_size'%path)
#bfReadAndSet(c, 'ColorOpacity', rd, '%s/opacity'%path)
#bfReadAndSet(c, 'GammaCorrection', rd, '%s/gamma'%path)
#bfReadAndSet(c, 'ColorRange', rd, '%s/range'%path)
# custom - any other tags in proprietary files should go further prefixed by the custom parent
custom = mee.xpath('sa:StructuredAnnotations/sa:XMLAnnotation', namespaces=namespaces)
for a in custom:
#k = misc.xpathtextnode(a, 'sa:Value/om:OriginalMetadata/om:Key', namespaces=namespaces) # dima: v4.x.x
#v = misc.xpathtextnode(a, 'sa:Value/om:OriginalMetadata/om:Value', namespaces=namespaces) # dima: v4.x.x
#k = misc.xpathtextnode(a, 'Value/OriginalMetadata/Key', namespaces=namespaces) # dima: v5.0.0
#v = misc.xpathtextnode(a, 'Value/OriginalMetadata/Value', namespaces=namespaces) # dima: v5.0.0
k = misc.xpathtextnode(a, 'sa:Value/sa:OriginalMetadata/sa:Key', namespaces=namespaces) # dima: v5.1.0
v = misc.xpathtextnode(a, 'sa:Value/sa:OriginalMetadata/sa:Value', namespaces=namespaces) # dima: v5.1.0
rd['custom/%s'%k] = v
return rd
def dryrun(self, token, arg):
arg = safeint(arg.lower(), 256) - 1
ofile = '%s.pixelcounter_%s.xml' % (token.data, arg)
return token.setXmlFile(fname=ofile)
def action(self, token, arg):
ss = arg.split(',')
size = [safeint(ss[0], 128) if len(ss)>0 else 128,
safeint(ss[1], 128) if len(ss)>1 else 128]
method = ss[2].upper() if len(ss)>2 and len(ss[2])>0 else 'BC'
preproc = ss[3].lower() if len(ss)>3 and len(ss[3])>0 else ''
preprocc = ',%s'%preproc if len(preproc)>0 else '' # attempt to keep the filename backward compatible
fmt = ss[4].lower() if len(ss)>4 and len(ss[4])>0 else 'jpeg'
if size[0]<=0 and size[1]<=0:
raise ImageServiceException(400, 'Thumbnail: size is unsupported [%s]'%arg)
if method not in ['NN', 'BL', 'BC']:
raise ImageServiceException(400, 'Thumbnail: method is unsupported [%s]'%arg)
if preproc not in ['', 'mid', 'mip', 'nip']:
raise ImageServiceException(400, 'Thumbnail: method is unsupported [%s]'%arg)
ext = self.server.converters.defaultExtension(fmt)
ifile = token.first_input_file()
ofile = '%s.thumb_%s,%s,%s%s.%s'%(token.data, size[0], size[1], method, preprocc, ext)
dims = token.dims or {}
num_x = int(dims.get('image_num_x', 0))
num_y = int(dims.get('image_num_y', 0))
try:
width, height = compute_new_size(num_x, num_y, size[0], size[1], keep_aspect_ratio=True, no_upsample=True)
except ZeroDivisionError:
raise ImageServiceException(400, 'Thumbnail: new image size cannot be guessed due to missing info' )
info = {
'image_num_x': width,
'image_num_y': height,
'image_num_c': 3,
'image_num_z': 1,
'image_num_t': 1,
'image_pixel_depth': 8,
'format': fmt,
}
# if image can be decoded with imageconvert, enqueue
if dims.get('converter', '') == ConverterImgcnv.name:
r = ConverterImgcnv.thumbnail(token, ofile, size[0], size[1], method=method, preproc=preproc, fmt=fmt)
if isinstance(r, list):
return self.server.enqueue(token, 'thumbnail', ofile, fmt=fmt, command=r, dims=info)
# if image requires other decoder
if not os.path.exists(ofile):
intermediate = '%s.ome.tif'%(token.data)
r = None
if 'converter' in dims and dims.get('converter') in self.server.converters:
r = self.server.converters[dims.get('converter')].thumbnail(token, ofile, size[0], size[1], method=method, intermediate=intermediate, preproc=preproc, fmt=fmt)
# if desired converter failed, perform exhaustive conversion
if r is None:
for n,c in self.server.converters.iteritems():
if n in [ConverterImgcnv.name, dims.get('converter')]: continue
r = c.thumbnail(token, ofile, size[0], size[1], method=method, intermediate=intermediate, preproc=preproc, fmt=fmt)
if r is not None:
break
if r is None:
log.error('Thumbnail %s: could not generate thumbnail for [%s]', token.resource_id, ifile)
raise ImageServiceException(415, 'Could not generate thumbnail' )
if isinstance(r, list):
return self.server.enqueue(token, 'thumbnail', ofile, fmt=fmt, command=r, dims=info)
return token.setImage(ofile, fmt=fmt, dims=info, input=ofile)
