def _create_pipeline_descriptions(self, value): if not isinstance(value, list): raise PipelineDescriptionError( 'Value of "pipeline" in pipeline description file must be an ' 'array.' ) description = list() for i, v in enumerate(value): if not isinstance(v, dict): raise PipelineDescriptionError( 'Value of item #%d in "pipeline" section of pipeline ' 'description must be a mapping.' ) # NOTE: The "name" attribute is only required for the viewer. if 'name' in v: module_name = v.pop('name') try: module = PipelineModuleDescription(**v) except TypeError as err: raise PipelineDescriptionError( 'Incorrect arguments provided for item #%d in "pipeline" ' 'section of pipeline description:\n%s' % (i, str(err)) ) description.append(module) return description
def version(self, value): if not isinstance(value, basestring): raise PipelineDescriptionError( 'Value of "version" in handles description must be a string.') if not re.search(r'^\d+\.\d+\.\d+$', value): raise PipelineDescriptionError( 'Value of "version" in handles description ' 'must follow semantic versioning.') self._version = str(value)
def _create_output_description(self, value): if not isinstance(value, dict): raise PipelineDescriptionError( 'Value of "output" in pipeline description file must be a ' 'mapping.') try: return PipelineOutputDescription(**value) except TypeError as err: raise PipelineDescriptionError( 'Incorrect arguments provided for "output" section of pipeline ' 'description:\n%s' % str(err))
def _create_object_descriptions(self, value): if not isinstance(value, list): raise PipelineDescriptionError( 'Value of "objects" in "input" section of pipeline descripion ' 'must be an array.') descriptions = list() for i, v in enumerate(value): if not isinstance(v, dict): raise PipelineDescriptionError( 'Value of item #%d of "objects" in "input" section of ' 'pipeline description must be a mapping.' % i) obj = PipelineObjectOutputDescription(**v) descriptions.append(obj) return descriptions
def _create_channel_descriptions(self, value): if not isinstance(value, list): raise PipelineDescriptionError( 'Value of "channels" in "input" section of pipeline descripion ' 'must be an array.') descriptions = list() for i, v in enumerate(value): if not isinstance(v, dict): raise PipelineDescriptionError( 'Value of item #%d of "channels" in "input" section of ' 'pipeline description must be a mapping.' % i) ch = PipelineChannelInputDescription(**v) descriptions.append(ch) return descriptions
def pipeline(self): '''List[tmlib.jterator.module.ImageAnalysisModule]: pipeline built based on :class`PipelineDescription <tmlib.workflow.jterator.description.PipelineDescription>` and :class:`HandleDescriptions <tmlib.workflow.jterator.description.HandleDescriptions>`. ''' handles = self.project.handles # only compute this once pipeline = list() for i, element in enumerate(self.project.pipe.description.pipeline): if not element.active: continue if '/' in element.source: logger.debug( 'assuming module `%s` resides outside the configured module path', element.source) source_file = os.path.expanduser( os.path.expandvars(element.source)) if not os.path.isabs(source_file): source_file = os.path.join(self.step_location, source_file) else: logger.debug( 'searching for module `%s` in configured module path %r ...', element.source, cfg.modules_path) source_file = AvailableModules().find_module_by_name( element.source) if not os.path.exists(source_file): raise PipelineDescriptionError( 'Module source `{0}` resolved to non-existing file `{1}`'. format(element.source, source_file)) pipeline.append( ImageAnalysisModule(name=handles[i].name, source_file=source_file, handles=handles[i].description)) return pipeline
def pipeline(self): '''List[tmlib.jterator.module.ImageAnalysisModule]: pipeline built based on :class`PipelineDescription <tmlib.workflow.jterator.description.PipelineDescription>` and :class:`HandleDescriptions <tmlib.workflow.jterator.description.HandleDescriptions>`. ''' pipeline = list() for i, element in enumerate(self.project.pipe.description.pipeline): if not element.active: continue source_file = element.source if '/' in source_file: source_file = os.path.expandvars(source_file) source_file = os.path.expanduser(source_file) if not os.path.isabs(source_file): source_file = os.path.join(self.step_location, source_file) if not os.path.exists(source_file): raise PipelineDescriptionError( 'Module file does not exist: %s' % source_file) name = self.project.handles[i].name handles = self.project.handles[i].description module = ImageAnalysisModule(name=name, source_file=source_file, handles=handles) pipeline.append(module) return pipeline
def _create_output_descriptions(self, value): if not isinstance(value, list): raise PipelineDescriptionError( 'Value of "output" in handles description must be an array.' ) names = list() descriptions = list() for i, v in enumerate(value): if not isinstance(v, dict): raise PipelineDescriptionError( 'Output item #%d in handles description must be a mapping.' % i ) if 'type' not in v: raise PipelineDescriptionError( 'Output item #%d in handles description requires key "type".' % i ) handle_type = v.pop('type') try: Handle = getattr(handles, handle_type) except AttributeError: raise PipelineDescriptionError( 'Type "%s" of output item #%d in handles description ' 'is not valid.' % (handle_type, i) ) try: h = Handle(**v) except TypeError as err: raise PipelineDescriptionError( 'Arguments for type "%s" of output item #%d ' 'in handles description are incorrect:\n%s' % (handle_type, i, str(err)) ) if not(isinstance(h, handles.OutputHandle) or isinstance(h, handles.PipeHandle)): raise PipelineDescriptionError( 'Type "%s" is not a valid output handle.' % handle_type ) descriptions.append(h) names.append(v['name']) if len(set(names)) < len(value): raise PipelineDescriptionError( 'Names of output items in handles description must be unique.' ) return descriptions
def _load_pipeline_input(self, site_id): logger.info('load pipeline inputs') # Use an in-memory store for pipeline data and only insert outputs # into the database once the whole pipeline has completed successfully. store = { 'site_id': site_id, 'pipe': dict(), 'current_figure': list(), 'objects': dict(), 'channels': list() } # Load the images, correct them if requested and align them if required. # NOTE: When the experiment was acquired in "multiplexing" mode, # images will be automatically aligned, assuming that this is the # desired behavior. channel_input = self.project.pipe.description.input.channels objects_input = self.project.pipe.description.input.objects with tm.utils.ExperimentSession(self.experiment_id) as session: site = session.query(tm.Site).get(site_id) y_offset, x_offset = site.aligned_offset height = site.aligned_height width = site.aligned_width for ch in channel_input: channel = session.query( tm.Channel.bit_depth, tm.Channel.id ).\ filter_by(name=ch.name).\ one() if channel.bit_depth == 16: dtype = np.uint16 elif channel.bit_depth == 8: dtype = np.uint8 records = session.query(tm.ChannelImageFile.tpoint).\ filter_by(site_id=site.id,channel_id=channel.id).\ distinct() tpoints = [r.tpoint for r in records] n_tpoints = len(tpoints) records = session.query(tm.ChannelImageFile.zplane).\ filter_by(site_id=site.id,channel_id=channel.id).\ distinct() zplanes = [r.zplane for r in records] n_zplanes = len(zplanes) image_array = np.zeros((height, width, n_zplanes, n_tpoints), dtype) if ch.correct: logger.info( 'load illumination statistics for channel "%s"', ch.name) try: stats_file = session.query(tm.IllumstatsFile).\ join(tm.Channel).\ filter(tm.Channel.name == ch.name).\ one() except NoResultFound: raise PipelineDescriptionError( 'No illumination statistics file found for ' 'channel "%s"' % ch.name) stats = stats_file.get() else: stats = None logger.info('load images for channel "%s"', ch.name) image_files = session.query(tm.ChannelImageFile).\ filter_by(site_id=site.id, channel_id=channel.id).\ all() for f in image_files: logger.info('load image %d', f.id) img = f.get() if ch.correct: logger.info('correct image %d', f.id) img = img.correct(stats) logger.debug('align image %d', f.id) img = img.align() # shifted and cropped! image_array[:, :, f.zplane, f.tpoint] = img.array store['pipe'][ch.name] = image_array for obj in objects_input: mapobject_type = session.query(tm.MapobjectType).\ filter_by(name=obj.name).\ one() polygons = list() for t in sorted(tpoints): zpolys = list() for z in sorted(zplanes): zpolys.append( mapobject_type.get_segmentations_per_site( site_id=site.id, tpoint=t, zplane=z)) polygons.append(zpolys) segm_obj = SegmentedObjects(obj.name, obj.name) segm_obj.add_polygons(polygons, y_offset, x_offset, (height, width)) store['objects'][segm_obj.name] = segm_obj store['pipe'][segm_obj.name] = segm_obj.value # Remove single-dimensions from image arrays. # NOTE: It would be more consistent to preserve shape, but most people # will work with 2D/3D images and having to deal with additional # dimensions would be rather annoying I assume. for name, img in store['pipe'].iteritems(): store['pipe'][name] = np.squeeze(img) return store