def dryrun(self, token, arg): ss = arg.split(',') size = [safeint(ss[0], 128) if len(ss)>0 else 128, safeint(ss[1], 128) if len(ss)>1 else 128] method = ss[2].upper() if len(ss)>2 and len(ss[2])>0 else 'BC' preproc = ss[3].lower() if len(ss)>3 and len(ss[3])>0 else '' preprocc = ',%s'%preproc if len(preproc)>0 else '' # attempt to keep the filename backward compatible fmt = ss[4].lower() if len(ss)>4 and len(ss[4])>0 else 'jpeg' dims = token.dims or {} num_x = int(dims.get('image_num_x', 0)) num_y = int(dims.get('image_num_y', 0)) try: width, height = compute_new_size(num_x, num_y, size[0], size[1], keep_aspect_ratio=True, no_upsample=True) except ZeroDivisionError: log.warning('Thumbnail warning while guessing size %s: [%sx%s] to [%sx%s]', token.resource_id, num_x, num_y, width, height) width = size[0] height = size[1] info = { 'image_num_x': width, 'image_num_y': height, 'image_num_c': 3, 'image_num_z': 1, 'image_num_t': 1, 'image_pixel_depth': 8, 'format': fmt, } ext = self.server.converters.defaultExtension(fmt) ofile = '%s.thumb_%s,%s,%s%s.%s'%(token.data, size[0], size[1], method, preprocc, ext) #log.debug('Dryrun thumbnail [%s]', ofile) return token.setImage(ofile, fmt=fmt, dims=info, input=ofile)
def action(self, token, arg): method = 'a' num_channels = 3 arg = arg.lower() if arg == 'display': args = ['-fusemeta'] argenc = 'display' elif arg == 'gray' or arg == 'grey': args = ['-fusegrey'] num_channels = 1 argenc = 'gray' else: args = ['-fusergb', arg] if ':' in arg: (arg, method) = arg.split(':', 1) elif '.' in arg: (arg, method) = arg.split('.', 1) argenc = ''.join([ hex(int(i)).replace('0x', '') for i in arg.replace(';', ',').split(',') if i is not '' ]) # test if all channels are valid dims = token.dims or {} img_num_c = misc.safeint(dims.get('image_num_c'), 0) groups = [i for i in arg.split(';') if i is not ''] # if there are more groups than input channels if len(groups) > img_num_c: groups = groups[img_num_c:] for i, g in enumerate(groups): channels = [misc.safeint(i, 0) for i in g.split(',')] # we can skip 0 weighted channels even if they are outside of the image channel range if max(channels) > 0: raise ImageServiceException( 400, 'Fuse: requested fusion of channel outside bounds %s>%s (%s)' % (img_num_c + i, img_num_c, g)) if method != 'a': args.extend(['-fusemethod', method]) info = { 'image_num_c': num_channels, } ifile = token.first_input_file() ofile = '%s.fuse_%s_%s' % (token.data, argenc, method) log.debug('Fuse %s: %s to %s with [%s:%s]', token.resource_id, ifile, ofile, arg, method) return self.server.enqueue(token, 'fuse', ofile, fmt=default_format, command=args, dims=info)
def tile(cls, token, ofnm, level, x, y, sz, **kw): '''extract tile Level,X,Y tile from input filename into output in OME-TIFF format''' # imgcnv driver does not support arbitrary size interface if kw.get('arbitrary_size', False) == True or level is None or sz is None: return None ifnm = token.first_input_file() series = token.series page = 0 log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series, ifnm) info = token.dims or {} tile_w = info.get('tile_num_x', 0) tile_h = info.get('tile_num_y', 0) num_l = info.get('image_num_resolution_levels', 1) if num_l <= 1 or tile_w < 1 or tile_h < 1: log.debug('Image does not contain tiles, skipping...') return None queue = token.getQueue() #command = ['-o', ofnm, '-t', 'tiff'] command = [] if token.series is not None and token.series != 0: command.extend(['-path', token.series]) # separate normal and multi-file series if '-i' not in queue and '-il' not in queue and '-page' not in queue: if token.is_multifile_series() is False: command.extend(['-i', ifnm]) command.extend(['-page', str(page + 1)]) else: # use first image of the series, need to check for separate channels here files = token.input meta = token.meta or {} samples = meta.get('image_num_c', 0) if samples < 2: command.extend(['-i', files[page]]) else: # in case of channels being stored in separate files page = page * samples command.extend(['-i', files[page]]) for s in range(1, samples): command.extend(['-c', files[page + s]]) level = misc.safeint(level, 0) x = misc.safeint(x, 0) y = misc.safeint(y, 0) sz = misc.safeint(sz, 0) command.extend(['-tile', '%s,%s,%s,%s' % (sz, x, y, level)]) # add speed file command.extend(['-speed', token.get_speed_file()]) return command
def get_formats(cls): '''inits supported file formats''' if cls.installed_formats is None: formats_xml = cls.run_command([cls.CONVERTERCOMMAND, '-fmtxml']) formats = etree.fromstring('<formats>%s</formats>' % formats_xml) cls.installed_formats = OrderedDict() codecs = formats.xpath('//codec') for c in codecs: try: name = c.get('name') fullname = c.xpath('tag[@name="fullname"]')[0].get( 'value', '') exts = c.xpath('tag[@name="extensions"]')[0].get( 'value', '').split('|') reading = len( c.xpath( 'tag[@name="support" and @value="reading"]')) > 0 writing = len( c.xpath( 'tag[@name="support" and @value="writing"]')) > 0 multipage = len( c.xpath( 'tag[@name="support" and @value="writing multiple pages"]' )) > 0 metadata = len( c.xpath( 'tag[@name="support" and @value="reading metadata"]' ) ) > 0 or len( c.xpath( 'tag[@name="support" and @value="writing metadata"]' )) > 0 samples_min = misc.safeint( c.xpath('tag[@name="min-samples-per-pixel"]')[0].get( 'value', '0')) samples_max = misc.safeint( c.xpath('tag[@name="max-samples-per-pixel"]')[0].get( 'value', '0')) bits_min = misc.safeint( c.xpath('tag[@name="min-bits-per-sample"]')[0].get( 'value', '0')) bits_max = misc.safeint( c.xpath('tag[@name="max-bits-per-sample"]')[0].get( 'value', '0')) except IndexError: continue cls.installed_formats[name.lower()] = Format( name=name, fullname=fullname, ext=exts, reading=reading, writing=writing, multipage=multipage, metadata=metadata, samples=(samples_min, samples_max), bits=(bits_min, bits_max))
def tile(cls, token, ofnm, level=None, x=None, y=None, sz=None, **kw): '''extract tile from image default interface: Level,X,Y tile from input filename into output in TIFF format alternative interface, not required to support and may return None in this case scale=scale, x1=x1, y1=y1, x2=x2, y2=y2, arbitrary_size=False ''' # open slide driver does not support arbitrary size interface if kw.get('arbitrary_size', False) == True or level is None or sz is None: return None ifnm = token.first_input_file() series = token.series if not cls.supported(token): return None log.debug('Tile: %s %s %s %s %s for [%s]', level, x, y, sz, series, ifnm) level = misc.safeint(level, 0) x = misc.safeint(x, 0) y = misc.safeint(y, 0) sz = misc.safeint(sz, 0) with Locks(ifnm, ofnm, failonexist=True) as l: if l.locked: # the file is not being currently written by another process try: _, tmp = misc.start_nounicode_win(ifnm, []) slide = openslide.OpenSlide(tmp or ifnm) dz = deepzoom.DeepZoomGenerator(slide, tile_size=sz, overlap=0) img = dz.get_tile(dz.level_count - level - 1, (x, y)) img.save(ofnm, 'TIFF', compression='LZW') slide.close() misc.end_nounicode_win(tmp) except (openslide.OpenSlideUnsupportedFormatError, openslide.OpenSlideError): misc.end_nounicode_win(tmp) return None # make sure the file was written with Locks(ofnm, failonread=(not block_reads)) as l: if l.locked is False: # dima: never wait, respond immediately raise ImageServiceFuture((1, 15)) return ofnm
def init_classes_dataset(self): if self.training_set is None: raise ConnoisseurException(responses.BAD_REQUEST, 'Cannot initialize classes due to missing training dataset') with Locks(None, self.lockable, failonexist=True) as l: if l.locked is False: # the file is being written by another process raise ConnoisseurException(responses.LOCKED, 'The model is locked for processing by another process') try: dataset_url = ensure_url(self.training_set) adapter_gobs = self.create_adapter_gobs(model=self, image=None) classes = {} gobs = data_service.query(resource_type='value', parent=dataset_url, extract='gobject[type]') idx = 0 self.total_samples = 0 for g in gobs: k = g.get('type') n = misc.safeint(g.text, 0) if k is None: continue k = adapter_gobs.get_class_name(g) # adapt the class name, might need some change since the node is not a true gobject if k is None: continue if k not in classes: classes[k] = { 'label': k, 'id': idx, 'samples': n, } idx += 1 else: classes[k]['samples'] += n self.total_samples += n self.classes_data = classes self.classes_data_by_original_id = dict((v['id'],v) for k,v in self.classes_data.iteritems()) #log.debug('Classes data: %s', str(self.classes_data)) self.classes_model = {} self.classes_model_by_id = {} self.classes_model_by_original_id = {} self.number_classes_in_model = 0 except: self.update_with_error('status.classes.init', 'Exception during init_classes_dataset') raise # update model resource set_tag(self.resource, 'total_samples', self.total_samples) set_classes(self.resource, 'classes_data', self.classes_data) set_classes(self.resource, 'classes_model', self.classes_model) set_tag(self.resource, 'status.classes.init', 'finished') self.sync_resource()
def action(self, token, arg): if not token.isFile(): raise ImageServiceException( 400, 'Pixelcounter: input is not an image...') arg = safeint(arg.lower(), 256) - 1 ifile = token.first_input_file() ofile = '%s.pixelcounter_%s.xml' % (token.data, arg) log.debug('Pixelcounter %s: %s to %s with [%s]', token.resource_id, ifile, ofile, arg) command = token.drainQueue() if not os.path.exists(ofile): command.extend(['-pixelcounts', str(arg)]) self.server.imageconvert(token, ifile, ofile, extra=command) return token.setXmlFile(fname=ofile)
def dryrun(self, token, arg): # parse arguments #log.debug('Dryrun Slice') x1 = 0 x2 = 0 y1 = 0 y2 = 0 z1 = 0 z2 = 0 t1 = 0 t2 = 0 extended_dims = {} extended_str = '' if ':' not in arg: # the standard way of defining dimensions vs = [[safeint(i, 0) for i in vs.split('-', 1)] for vs in arg.split(',')] for v in vs: if len(v) < 2: v.append(0) for v in range(len(vs) - 4): vs.append([0, 0]) x1, x2 = vs[0] y1, y2 = vs[1] z1, z2 = vs[2] t1, t2 = vs[3] else: # the extended way of defining dimensions for a in arg.split(','): k, v = a.split(':', 1) if k not in self.dimension_names: continue extended_dims[k] = [safeint(i, 0) for i in v.split('-', 1)] # update x,y,z and t from the new style args x1, x2 = pop_dimension_range(extended_dims, 'x') y1, y2 = pop_dimension_range(extended_dims, 'y') z1, z2 = pop_dimension_range(extended_dims, 'z') t1, t2 = pop_dimension_range(extended_dims, 't') # update extended string part if len(extended_dims) > 0: # get a string of extended dimensions in the defined order ddd = [ get_dimension_string(extended_dims, k) for k in self.dimension_names if k in extended_dims ] extended_str = '.%s' % (','.join(ddd)) # in case slices request an exact copy, skip if x1 == 0 and x2 == 0 and y1 == 0 and y2 == 0 and z1 == 0 and z2 == 0 and t1 == 0 and t2 == 0 and len( extended_dims) < 1: #log.debug('Dryrun Slice: skipping due to pars==0') return token dims = token.dims or {} if x1 <= 1 and x2 >= dims.get('image_num_x', 1): x1 = 0 x2 = 0 if y1 <= 1 and y2 >= dims.get('image_num_y', 1): y1 = 0 y2 = 0 if z1 <= 1 and z2 >= dims.get('image_num_z', 1): z1 = 0 z2 = 0 if t1 <= 1 and t2 >= dims.get('image_num_t', 1): t1 = 0 t2 = 0 # check image bounds - dima: comment since dryrun may not contain proper dimensions # if z1>dims.get('image_num_z', 1): # raise ImageServiceException(400, 'Slice: requested Z slice outside of image bounds: [%s]'%z1 ) # if z2>dims.get('image_num_z', 1): # raise ImageServiceException(400, 'Slice: requested Z slice outside of image bounds: [%s]'%z2 ) # if t1>dims.get('image_num_t', 1): # raise ImageServiceException(400, 'Slice: requested T plane outside of image bounds: [%s]'%t1 ) # if t2>dims.get('image_num_t', 1): # raise ImageServiceException(400, 'Slice: requested T plane outside of image bounds: [%s]'%t2 ) # shortcuts are only possible with no ROIs are requested if x1 == x2 == 0 and y1 == y2 == 0 and len(dims) > 0: # shortcut if input image has only one T and Z if dims.get('image_num_z', 1) <= 1 and dims.get( 'image_num_t', 1) <= 1 and len(extended_dims) < 1: #log.debug('Dryrun Slice: plane requested on image with no T or Z planes, skipping...') return token # shortcut if asking for all slices with only a specific time point in an image with only one time pont if z1 == z2 == 0 and t1 <= 1 and dims.get( 'image_num_t', 1) <= 1 and len(extended_dims) < 1: #log.debug('Dryrun Slice: T plane requested on image with no T planes, skipping...') return token # shortcut if asking for all time points with only a specific z slice in an image with only one z slice if t1 == t2 == 0 and z1 <= 1 and dims.get( 'image_num_z', 1) <= 1 and len(extended_dims) < 1: #log.debug('Dryrun Slice: Z plane requested on image with no Z planes, skipping...') return token if z1 == z2 == 0: z1 = 1 z2 = dims.get('image_num_z', 1) if t1 == t2 == 0: t1 = 1 t2 = dims.get('image_num_t', 1) new_w = x2 - x1 new_h = y2 - y1 new_z = max(1, z2 - z1 + 1) new_t = max(1, t2 - t1 + 1) info = { 'image_num_z': new_z, 'image_num_t': new_t, } if new_w > 0: info['image_num_x'] = new_w + 1 if new_h > 0: info['image_num_y'] = new_h + 1 ofname = '%s.%d-%d,%d-%d,%d-%d,%d-%d%s.ome.tif' % ( token.data, x1, x2, y1, y2, z1, z2, t1, t2, extended_str) #log.debug('Dryrun Slice [%s]', ofname) return token.setImage(ofname, fmt=default_format, dims=info, input=ofname)
def action(self, token, arg): '''arg = x1-x2,y1-y2,z|z1-z2,t|t1-t2 or arg = z:V1-V2,fov:V,... ''' if not token.isFile(): raise ImageServiceException(400, 'Slice: input is not an image...') # parse arguments x1 = 0 x2 = 0 y1 = 0 y2 = 0 z1 = 0 z2 = 0 t1 = 0 t2 = 0 extended_dims = {} extended_str = '' if ':' not in arg: # the standard way of defining dimensions vs = [[safeint(i, 0) for i in vs.split('-', 1)] for vs in arg.split(',')] for v in vs: if len(v) < 2: v.append(0) for v in range(len(vs) - 4): vs.append([0, 0]) x1, x2 = vs[0] y1, y2 = vs[1] z1, z2 = vs[2] t1, t2 = vs[3] else: # the extended way of defining dimensions for a in arg.split(','): k, v = a.split(':', 1) if k not in self.dimension_names: continue extended_dims[k] = [safeint(i, 0) for i in v.split('-', 1)] # update x,y,z and t from the new style args x1, x2 = pop_dimension_range(extended_dims, 'x') y1, y2 = pop_dimension_range(extended_dims, 'y') z1, z2 = pop_dimension_range(extended_dims, 'z') t1, t2 = pop_dimension_range(extended_dims, 't') # update extended string part if len(extended_dims) > 0: # get a string of extended dimensions in the defined order ddd = [ get_dimension_string(extended_dims, k) for k in self.dimension_names if k in extended_dims ] extended_str = '.%s' % (','.join(ddd)) # in case slices request an exact copy, skip if x1 == 0 and x2 == 0 and y1 == 0 and y2 == 0 and z1 == 0 and z2 == 0 and t1 == 0 and t2 == 0 and len( extended_dims) < 1: return token dims = token.dims or {} if x1 <= 1 and x2 >= dims.get('image_num_x', 1): x1 = 0 x2 = 0 if y1 <= 1 and y2 >= dims.get('image_num_y', 1): y1 = 0 y2 = 0 if z1 <= 1 and z2 >= dims.get('image_num_z', 1): z1 = 0 z2 = 0 if t1 <= 1 and t2 >= dims.get('image_num_t', 1): t1 = 0 t2 = 0 # check image bounds if z1 > dims.get('image_num_z', 1): raise ImageServiceException( 400, 'Slice: requested Z slice outside of image bounds: [%s]' % z1) if z2 > dims.get('image_num_z', 1): raise ImageServiceException( 400, 'Slice: requested Z slice outside of image bounds: [%s]' % z2) if t1 > dims.get('image_num_t', 1): raise ImageServiceException( 400, 'Slice: requested T plane outside of image bounds: [%s]' % t1) if t2 > dims.get('image_num_t', 1): raise ImageServiceException( 400, 'Slice: requested T plane outside of image bounds: [%s]' % t2) # shortcuts are only possible with no ROIs are requested if x1 == x2 == 0 and y1 == y2 == 0: # shortcut if input image has only one T and Z if dims.get('image_num_z', 1) <= 1 and dims.get( 'image_num_t', 1) <= 1 and len(extended_dims) < 1: log.debug( 'Slice: plane requested on image with no T or Z planes, skipping...' ) return token # shortcut if asking for all slices with only a specific time point in an image with only one time pont if z1 == z2 == 0 and t1 <= 1 and dims.get( 'image_num_t', 1) <= 1 and len(extended_dims) < 1: log.debug( 'Slice: T plane requested on image with no T planes, skipping...' ) return token # shortcut if asking for all time points with only a specific z slice in an image with only one z slice if t1 == t2 == 0 and z1 <= 1 and dims.get( 'image_num_z', 1) <= 1 and len(extended_dims) < 1: log.debug( 'Slice: Z plane requested on image with no Z planes, skipping...' ) return token if z1 == z2 == 0: z1 = 1 z2 = dims.get('image_num_z', 1) if t1 == t2 == 0: t1 = 1 t2 = dims.get('image_num_t', 1) # construct a sliced filename ifname = token.first_input_file() ofname = '%s.%d-%d,%d-%d,%d-%d,%d-%d%s.ome.tif' % ( token.data, x1, x2, y1, y2, z1, z2, t1, t2, extended_str) log.debug('Slice %s: from [%s] to [%s]', token.resource_id, ifname, ofname) new_w = x2 - x1 new_h = y2 - y1 new_z = max(1, z2 - z1 + 1) new_t = max(1, t2 - t1 + 1) info = { 'image_num_z': new_z, 'image_num_t': new_t, } if new_w > 0: info['image_num_x'] = new_w + 1 if new_h > 0: info['image_num_y'] = new_h + 1 meta = token.meta or {} unsupported_multifile = False if token.is_multifile_series() is True and (z2 == 0 or z2 == z1) and ( t2 == 0 or t2 == t1) and x1 == x2 and y1 == y2 and meta.get( 'image_num_c', 0) == 0: unsupported_multifile = True if dims.get( 'converter', '') == ConverterImgcnv.name or unsupported_multifile is True: r = ConverterImgcnv.slice(token, ofname, z=(z1, z2), t=(t1, t2), roi=(x1, x2, y1, y2), fmt=default_format, **extended_dims) # if decoder returned a list of operations for imgcnv to enqueue if isinstance(r, list): return self.server.enqueue(token, 'slice', ofname, fmt=default_format, command=r, dims=info) # slice the image if not os.path.exists(ofname): intermediate = '%s.ome.tif' % token.data if 'converter' in dims and dims.get( 'converter') in self.server.converters: r = self.server.converters[dims.get('converter')].slice( token, ofname, z=(z1, z2), t=(t1, t2), roi=(x1, x2, y1, y2), fmt=default_format, intermediate=intermediate) # if desired converter failed, perform exhaustive conversion if r is None: for n, c in self.server.converters.iteritems(): if n in [ConverterImgcnv.name, dims.get('converter')]: continue r = c.slice(token, ofname, z=(z1, z2), t=(t1, t2), roi=(x1, x2, y1, y2), fmt=default_format, intermediate=intermediate) if r is not None: break if r is None: log.error('Slice %s: could not generate slice for [%s]', token.resource_id, ifname) raise ImageServiceException(415, 'Could not generate slice') # if decoder returned a list of operations for imgcnv to enqueue if isinstance(r, list): return self.server.enqueue(token, 'slice', ofname, fmt=default_format, command=r, dims=info) return token.setImage(ofname, fmt=default_format, dims=info, input=ofname)
def meta(cls, token, **kw): if not cls.installed: return {} ifnm = token.first_input_file() series = int(token.series) log.debug('Meta for: %s', ifnm ) if os.name == 'nt': nulldevice = 'NUL' elif os.name == 'posix': nulldevice = '/dev/null' command = [cls.CONVERTERCOMMAND, '-i', ifnm, '-m', '-l', nulldevice, '-ii', '%s'%series] command.extend( cls.extension(token, **kw) ) # extend if timeout or meta are present meta = cls.run_read(ifnm, command) if meta is None: return {} # fix a bug in Imaris Convert exporting XML with invalid chars # by removing the <ImplParameters> tag # params is formatted in INI format params = '' try: p = misc.between(BLOCK_START, BLOCK_END, meta) meta = meta.replace('%s%s%s'%(BLOCK_START, p, BLOCK_END), '', 1) params = p except UnboundLocalError: return {} ######################################## # Parse Meta XML # most files have improper encodings, try to recover ######################################## rd = {} try: mee = etree.fromstring(meta) except etree.XMLSyntaxError: try: mee = etree.fromstring(meta, parser=etree.XMLParser(encoding='iso-8859-1')) except (etree.XMLSyntaxError, LookupError): try: mee = etree.fromstring(meta, parser=etree.XMLParser(encoding='utf-16')) except (etree.XMLSyntaxError, LookupError): try: mee = etree.fromstring(meta, parser=etree.XMLParser(recover=True)) except etree.XMLSyntaxError: log.error ("Unparsable %s", meta) return {} if '<FileInfo2>' in meta: # v7 rd['image_num_series'] = misc.safeint(misc.xpathtextnode(mee, '/FileInfo2/NumberOfImages'), 1) imagenodepath = '/FileInfo2/Image[@mIndex="%s"]'%series else: # v8 rd['image_num_series'] = misc.safeint(misc.xpathtextnode(mee, '/MetaData/NumberOfImages'), 1) imagenodepath = '/MetaData/Image[@mIndex="%s"]'%series rd['image_series_index'] = series rd['date_time'] = misc.xpathtextnode(mee, '%s/ImplTimeInfo'%imagenodepath).split(';', 1)[0] #rd['format'] = misc.xpathtextnode(mee, '%s/BaseDescription'%imagenodepath).split(':', 1)[1].strip(' ') rd['format'] = misc.xpathtextnode(mee, '%s/BaseDescription'%imagenodepath) #.split(':', 1)[1].strip(' ') # dims dims = misc.xpathtextnode(mee, '%s/BaseDimension'%imagenodepath).split(' ') try: rd['image_num_x'] = misc.safeint(dims[0]) rd['image_num_y'] = misc.safeint(dims[1]) rd['image_num_z'] = misc.safeint(dims[2]) rd['image_num_c'] = misc.safeint(dims[3]) rd['image_num_t'] = misc.safeint(dims[4]) except IndexError: pass # pixel format pixeltypes = { 'uint8': ('unsigned integer', 8), 'uint16': ('unsigned integer', 16), 'uint32': ('unsigned integer', 32), 'uint64': ('unsigned integer', 64), 'int8': ('signed integer', 8), 'int16': ('signed integer', 16), 'int32': ('signed integer', 32), 'int64': ('signed integer', 64), 'float': ('floating point', 32), 'double': ('floating point', 64), } try: t = pixeltypes[misc.xpathtextnode(mee, '%s/ImplDataType'%imagenodepath).lower()] rd['image_pixel_format'] = t[0] rd['image_pixel_depth'] = t[1] except KeyError: pass # resolution extmin = [misc.safefloat(i) for i in misc.xpathtextnode(mee, '%s/ImplExtendMin'%imagenodepath).split(' ')] extmax = [misc.safefloat(i) for i in misc.xpathtextnode(mee, '%s/ImplExtendMax'%imagenodepath).split(' ')] rd['pixel_resolution_x'] = (extmax[0]-extmin[0])/rd['image_num_x'] rd['pixel_resolution_y'] = (extmax[1]-extmin[1])/rd['image_num_y'] rd['pixel_resolution_z'] = (extmax[2]-extmin[2])/rd['image_num_z'] # Time resolution is apparently missing in Imaris XML #rd['pixel_resolution_z'] = (extmax[2]-extmin[2])/rd['image_num_z'] rd['pixel_resolution_unit_x'] = 'microns' rd['pixel_resolution_unit_y'] = 'microns' rd['pixel_resolution_unit_z'] = 'microns' ######################################## # Parse params INI ######################################## #params = misc.xpathtextnode(mee, '%s/ImplParameters'%imagenodepath) # use index 0 since we fetch meta data with imageindex argument sp = StringIO.StringIO(params) config = ConfigParser.ConfigParser() try: config.readfp(sp) except Exception: safe_config_read(config, sp) sp.close() # custom - any tags in proprietary files should go further prefixed by the custom parent for section in config.sections(): for option in config.options(section): rd['custom/%s/%s'%(section,option)] = config.get(section, option) # Image parameters safeReadAndSet(config, 'Image', 'numericalaperture', rd, 'numerical_aperture') # channel names, colors and other info for c in range(rd['image_num_c']): section = 'Channel %s'%c path = 'channels/channel_%.5d'%c name = safeRead(config, section, 'Name') or '' dye = safeRead(config, section, 'Dye name') or safeRead(config, section, 'Fluor') if dye is not None: name = '%s (%s)'%(name, dye) if len(name)>0: rd['channel_%s_name'%c] = name rgb = safeRead(config, section, 'Color') if rgb is not None: rd['channel_color_%s'%c] = ','.join([str(int(misc.safefloat(i)*255)) for i in rgb.split(' ')]) # new channel format if len(name)>0: rd['%s/name'%path] = name if rgb is not None: rd['%s/color'%path] = ','.join(rgb.split(' ')) safeReadAndSet(config, section, 'ColorOpacity', rd, '%s/opacity'%path) safeReadAndSet(config, section, 'Fluor', rd, '%s/fluor'%path) safeReadAndSet(config, section, 'GammaCorrection', rd, '%s/gamma'%path) safeReadAndSet(config, section, 'LSMEmissionWavelength', rd, '%s/lsm_emission_wavelength'%path) safeReadAndSet(config, section, 'LSMExcitationWavelength', rd, '%s/lsm_excitation_wavelength'%path) safeReadAndSet(config, section, 'LSMPinhole', rd, '%s/lsm_pinhole_radius'%path) safeReadAndSet(config, section, 'objective', rd, '%s/objective'%path) rng = safeRead(config, section, 'ColorRange') if rng is not None: rd['%s/range'%path] = ','.join(rng.split(' ')) # read Zeiss CZI specific metadata not parsed properly by the Imaris convert qpath = 'custom/ZeissAttrs/imagedocument/metadata/information/image/dimensions/channels/channel' # color as defined in CZI and not properly red by ImarisConvert t = '%s/color %s'%(qpath, c) if t in rd: #<tag name="color 0" value="#FF0000FF"/> try: v = rd[t] r,g,b,a = imaris_to_rgb(v) #new rd['%s/color'%path] = '%s,%s,%s'%(r/255.0, g/255.0, b/255.0) # old rd['channel_color_%s'%c] = '%s,%s,%s'%(r, g, b) except Exception: pass # channel exposure defined in CZI t = '%s/exposuretime %s'%(qpath, c) if t in rd: try: #<tag name="exposuretime 0" value="53000000"/> -> 53.0 ms v = misc.safefloat(rd[t]) rd['%s/exposure'%path] = v/1000000.0 rd['%s/exposure_units'%path] = 'ms' except Exception: pass # instrument and assay names, not standard CZI but used in industry path = 'custom/Document' qpath = 'custom/ZeissAttrs/imagedocument/metadata/experiment/experimentblocks/acquisitionblock/processinggraph/filters/filter' t = '%s/argument'%(qpath) if t in rd: #"INSTRUMENT_NAME:Axio2;ASSAY_NAME:ARv7;FILTER_NAME:Epic.AnalysisFilters.ARv7.dll" v = rd[t] try: v = [i.split(':') for i in v.split(';')] for i in v: rd['%s/%s'%(path, i[0].lower())] = i[1] except Exception: pass # slide id found in barcode path = 'custom/Document' qpath = 'custom/ZeissAttrs/imagedocument/metadata/attachmentinfos/attachmentinfo/label/barcodes/barcode' t = '%s/content'%(qpath) if t in rd: rd['%s/slide_id'%path] = rd[t] return rd
def meta(cls, token, **kw): if not cls.installed: return {} ifnm = token.first_input_file() series = token.series if not os.path.exists(ifnm): return {} log.debug('Meta for: %s', ifnm ) o = cls.run_read(ifnm, [cls.BFINFO, '-nopix', '-omexml', '-novalid', '-no-upgrade', '-series', '%s'%series, ifnm] ) if o is None: return {} # extract the OME-XML part try: omexml = misc.between('<OME', '</OME>', o) omexml = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n<OME%s</OME>'%omexml except UnboundLocalError: return {} ######################################## # Parse non XML parts ######################################## rd = {} rd['image_num_series'] = misc.safeint(misc.between('Series count =', '\n', o),1) rd['image_series_index'] = series rd['format'] = misc.between('Checking file format [', ']', o) ######################################## # Parse Meta XML # bioformats defines UTF-8 encoding, but it may be something else, try to recover ######################################## try: mee = etree.fromstring(omexml) except etree.XMLSyntaxError: try: mee = etree.fromstring(omexml, parser=etree.XMLParser(encoding='iso-8859-1')) except (etree.XMLSyntaxError, LookupError): try: mee = etree.fromstring(omexml, parser=etree.XMLParser(encoding='cp1252')) except (etree.XMLSyntaxError, LookupError): mee = etree.fromstring(omexml, parser=etree.XMLParser(recover=True)) imagenodepath = 'ome:Image[@ID="Image:%s"]'%(series) namespaces = { 'ome': misc.between('OME xmlns="', '"', omexml), 'sa': misc.between('StructuredAnnotations xmlns="', '"', omexml), #'om': misc.between('OriginalMetadata xmlns="', '"', o), # dima: v4.x.x } rd['date_time'] = misc.xpathtextnode(mee, '%s/ome:AcquisitionDate'%imagenodepath, namespaces=namespaces).replace('T', ' ') pixels = mee.xpath('%s/ome:Pixels'%imagenodepath, namespaces=namespaces)[0] rd['image_num_x'] = misc.safeint(pixels.get('SizeX', '0').split(' ')[0]) rd['image_num_y'] = misc.safeint(pixels.get('SizeY', '0').split(' ')[0]) rd['image_num_z'] = misc.safeint(pixels.get('SizeZ', '0').split(' ')[0]) rd['image_num_c'] = misc.safeint(pixels.get('SizeC', '0').split(' ')[0]) # fix for '3 (effectively 1)' as number of channels rd['image_num_t'] = misc.safeint(pixels.get('SizeT', '0').split(' ')[0]) # pixel format pixeltypes = { 'uint8': ('unsigned integer', 8), 'uint16': ('unsigned integer', 16), 'uint32': ('unsigned integer', 32), 'uint64': ('unsigned integer', 64), 'int8': ('signed integer', 8), 'int16': ('signed integer', 16), 'int32': ('signed integer', 32), 'int64': ('signed integer', 64), 'float': ('floating point', 32), 'double': ('floating point', 64), } try: t = pixeltypes[pixels.get('Type', 0).lower()] rd['image_pixel_format'] = t[0] rd['image_pixel_depth'] = t[1] except KeyError: pass # resolution rd['pixel_resolution_x'] = misc.safefloat(pixels.get('PhysicalSizeX', '0.0')) rd['pixel_resolution_y'] = misc.safefloat(pixels.get('PhysicalSizeY', '0.0')) rd['pixel_resolution_z'] = misc.safefloat(pixels.get('PhysicalSizeZ', '0.0')) rd['pixel_resolution_t'] = misc.safefloat(pixels.get('TimeIncrement', '0.0')) rd['pixel_resolution_unit_x'] = 'microns' rd['pixel_resolution_unit_y'] = 'microns' rd['pixel_resolution_unit_z'] = 'microns' rd['pixel_resolution_unit_t'] = 'seconds' # default channel mapping if rd.get('image_num_c', 0) == 1: rd.setdefault('channel_color_0', '255,255,255') # channel info channels = mee.xpath('%s/ome:Pixels/ome:Channel'%imagenodepath, namespaces=namespaces) for c,i in zip(channels, range(len(channels))): bfReadAndSet(c, 'Name', rd, 'channel_%s_name'%i, defval='ch_%s'%i) bfReadAndSet(c, 'Color', rd, 'channel_color_%s'%i, f=bfColorToString) # new format for c,i in zip(channels, range(len(channels))): path = 'channels/channel_%.5d'%i bfReadAndSet(c, 'Name', rd, '%s/name'%path, defval='ch_%s'%i) bfReadAndSet(c, 'Color', rd, '%s/color'%path, f=bfColorToString) bfReadAndSet(c, 'Fluor', rd, '%s/fluor'%path) bfReadAndSet(c, 'EmissionWavelength', rd, '%s/lsm_emission_wavelength'%path) bfReadAndSet(c, 'ExcitationWavelength', rd, '%s/lsm_excitation_wavelength'%path) bfReadAndSet(c, 'ContrastMethod', rd, '%s/contrast_method'%path) bfReadAndSet(c, 'AcquisitionMode', rd, '%s/acquisition_mode'%path) bfReadAndSet(c, 'IlluminationType', rd, '%s/illumination'%path) bfReadAndSet(c, 'PinholeSize', rd, '%s/pinhole_size'%path) #bfReadAndSet(c, 'ColorOpacity', rd, '%s/opacity'%path) #bfReadAndSet(c, 'GammaCorrection', rd, '%s/gamma'%path) #bfReadAndSet(c, 'ColorRange', rd, '%s/range'%path) # custom - any other tags in proprietary files should go further prefixed by the custom parent custom = mee.xpath('sa:StructuredAnnotations/sa:XMLAnnotation', namespaces=namespaces) for a in custom: #k = misc.xpathtextnode(a, 'sa:Value/om:OriginalMetadata/om:Key', namespaces=namespaces) # dima: v4.x.x #v = misc.xpathtextnode(a, 'sa:Value/om:OriginalMetadata/om:Value', namespaces=namespaces) # dima: v4.x.x #k = misc.xpathtextnode(a, 'Value/OriginalMetadata/Key', namespaces=namespaces) # dima: v5.0.0 #v = misc.xpathtextnode(a, 'Value/OriginalMetadata/Value', namespaces=namespaces) # dima: v5.0.0 k = misc.xpathtextnode(a, 'sa:Value/sa:OriginalMetadata/sa:Key', namespaces=namespaces) # dima: v5.1.0 v = misc.xpathtextnode(a, 'sa:Value/sa:OriginalMetadata/sa:Value', namespaces=namespaces) # dima: v5.1.0 rd['custom/%s'%k] = v return rd
def dryrun(self, token, arg): arg = safeint(arg.lower(), 256) - 1 ofile = '%s.pixelcounter_%s.xml' % (token.data, arg) return token.setXmlFile(fname=ofile)
def action(self, token, arg): ss = arg.split(',') size = [safeint(ss[0], 128) if len(ss)>0 else 128, safeint(ss[1], 128) if len(ss)>1 else 128] method = ss[2].upper() if len(ss)>2 and len(ss[2])>0 else 'BC' preproc = ss[3].lower() if len(ss)>3 and len(ss[3])>0 else '' preprocc = ',%s'%preproc if len(preproc)>0 else '' # attempt to keep the filename backward compatible fmt = ss[4].lower() if len(ss)>4 and len(ss[4])>0 else 'jpeg' if size[0]<=0 and size[1]<=0: raise ImageServiceException(400, 'Thumbnail: size is unsupported [%s]'%arg) if method not in ['NN', 'BL', 'BC']: raise ImageServiceException(400, 'Thumbnail: method is unsupported [%s]'%arg) if preproc not in ['', 'mid', 'mip', 'nip']: raise ImageServiceException(400, 'Thumbnail: method is unsupported [%s]'%arg) ext = self.server.converters.defaultExtension(fmt) ifile = token.first_input_file() ofile = '%s.thumb_%s,%s,%s%s.%s'%(token.data, size[0], size[1], method, preprocc, ext) dims = token.dims or {} num_x = int(dims.get('image_num_x', 0)) num_y = int(dims.get('image_num_y', 0)) try: width, height = compute_new_size(num_x, num_y, size[0], size[1], keep_aspect_ratio=True, no_upsample=True) except ZeroDivisionError: raise ImageServiceException(400, 'Thumbnail: new image size cannot be guessed due to missing info' ) info = { 'image_num_x': width, 'image_num_y': height, 'image_num_c': 3, 'image_num_z': 1, 'image_num_t': 1, 'image_pixel_depth': 8, 'format': fmt, } # if image can be decoded with imageconvert, enqueue if dims.get('converter', '') == ConverterImgcnv.name: r = ConverterImgcnv.thumbnail(token, ofile, size[0], size[1], method=method, preproc=preproc, fmt=fmt) if isinstance(r, list): return self.server.enqueue(token, 'thumbnail', ofile, fmt=fmt, command=r, dims=info) # if image requires other decoder if not os.path.exists(ofile): intermediate = '%s.ome.tif'%(token.data) r = None if 'converter' in dims and dims.get('converter') in self.server.converters: r = self.server.converters[dims.get('converter')].thumbnail(token, ofile, size[0], size[1], method=method, intermediate=intermediate, preproc=preproc, fmt=fmt) # if desired converter failed, perform exhaustive conversion if r is None: for n,c in self.server.converters.iteritems(): if n in [ConverterImgcnv.name, dims.get('converter')]: continue r = c.thumbnail(token, ofile, size[0], size[1], method=method, intermediate=intermediate, preproc=preproc, fmt=fmt) if r is not None: break if r is None: log.error('Thumbnail %s: could not generate thumbnail for [%s]', token.resource_id, ifile) raise ImageServiceException(415, 'Could not generate thumbnail' ) if isinstance(r, list): return self.server.enqueue(token, 'thumbnail', ofile, fmt=fmt, command=r, dims=info) return token.setImage(ofile, fmt=fmt, dims=info, input=ofile)