def _create_pipeline_descriptions(self, value):
if not isinstance(value, list):
raise PipelineDescriptionError(
'Value of "pipeline" in pipeline description file must be an '
'array.'
)
description = list()
for i, v in enumerate(value):
if not isinstance(v, dict):
raise PipelineDescriptionError(
'Value of item #%d in "pipeline" section of pipeline '
'description must be a mapping.'
)
# NOTE: The "name" attribute is only required for the viewer.
if 'name' in v:
module_name = v.pop('name')
try:
module = PipelineModuleDescription(**v)
except TypeError as err:
raise PipelineDescriptionError(
'Incorrect arguments provided for item #%d in "pipeline" '
'section of pipeline description:\n%s' % (i, str(err))
)
description.append(module)
return description
def version(self, value):
if not isinstance(value, basestring):
raise PipelineDescriptionError(
'Value of "version" in handles description must be a string.')
if not re.search(r'^\d+\.\d+\.\d+$', value):
raise PipelineDescriptionError(
'Value of "version" in handles description '
'must follow semantic versioning.')
self._version = str(value)
def _create_output_description(self, value):
if not isinstance(value, dict):
raise PipelineDescriptionError(
'Value of "output" in pipeline description file must be a '
'mapping.')
try:
return PipelineOutputDescription(**value)
except TypeError as err:
raise PipelineDescriptionError(
'Incorrect arguments provided for "output" section of pipeline '
'description:\n%s' % str(err))
def _create_object_descriptions(self, value):
if not isinstance(value, list):
raise PipelineDescriptionError(
'Value of "objects" in "input" section of pipeline descripion '
'must be an array.')
descriptions = list()
for i, v in enumerate(value):
if not isinstance(v, dict):
raise PipelineDescriptionError(
'Value of item #%d of "objects" in "input" section of '
'pipeline description must be a mapping.' % i)
obj = PipelineObjectOutputDescription(**v)
descriptions.append(obj)
return descriptions
def _create_channel_descriptions(self, value):
if not isinstance(value, list):
raise PipelineDescriptionError(
'Value of "channels" in "input" section of pipeline descripion '
'must be an array.')
descriptions = list()
for i, v in enumerate(value):
if not isinstance(v, dict):
raise PipelineDescriptionError(
'Value of item #%d of "channels" in "input" section of '
'pipeline description must be a mapping.' % i)
ch = PipelineChannelInputDescription(**v)
descriptions.append(ch)
return descriptions
def pipeline(self):
'''List[tmlib.jterator.module.ImageAnalysisModule]:
pipeline built based on
:class`PipelineDescription <tmlib.workflow.jterator.description.PipelineDescription>` and
:class:`HandleDescriptions <tmlib.workflow.jterator.description.HandleDescriptions>`.
'''
handles = self.project.handles # only compute this once
pipeline = list()
for i, element in enumerate(self.project.pipe.description.pipeline):
if not element.active:
continue
if '/' in element.source:
logger.debug(
'assuming module `%s` resides outside the configured module path',
element.source)
source_file = os.path.expanduser(
os.path.expandvars(element.source))
if not os.path.isabs(source_file):
source_file = os.path.join(self.step_location, source_file)
else:
logger.debug(
'searching for module `%s` in configured module path %r ...',
element.source, cfg.modules_path)
source_file = AvailableModules().find_module_by_name(
element.source)
if not os.path.exists(source_file):
raise PipelineDescriptionError(
'Module source `{0}` resolved to non-existing file `{1}`'.
format(element.source, source_file))
pipeline.append(
ImageAnalysisModule(name=handles[i].name,
source_file=source_file,
handles=handles[i].description))
return pipeline
def pipeline(self):
'''List[tmlib.jterator.module.ImageAnalysisModule]:
pipeline built based on
:class`PipelineDescription <tmlib.workflow.jterator.description.PipelineDescription>` and
:class:`HandleDescriptions <tmlib.workflow.jterator.description.HandleDescriptions>`.
'''
pipeline = list()
for i, element in enumerate(self.project.pipe.description.pipeline):
if not element.active:
continue
source_file = element.source
if '/' in source_file:
source_file = os.path.expandvars(source_file)
source_file = os.path.expanduser(source_file)
if not os.path.isabs(source_file):
source_file = os.path.join(self.step_location, source_file)
if not os.path.exists(source_file):
raise PipelineDescriptionError(
'Module file does not exist: %s' % source_file)
name = self.project.handles[i].name
handles = self.project.handles[i].description
module = ImageAnalysisModule(name=name,
source_file=source_file,
handles=handles)
pipeline.append(module)
return pipeline
def _create_output_descriptions(self, value):
if not isinstance(value, list):
raise PipelineDescriptionError(
'Value of "output" in handles description must be an array.'
)
names = list()
descriptions = list()
for i, v in enumerate(value):
if not isinstance(v, dict):
raise PipelineDescriptionError(
'Output item #%d in handles description must be a mapping.'
% i
)
if 'type' not in v:
raise PipelineDescriptionError(
'Output item #%d in handles description requires key "type".'
% i
)
handle_type = v.pop('type')
try:
Handle = getattr(handles, handle_type)
except AttributeError:
raise PipelineDescriptionError(
'Type "%s" of output item #%d in handles description '
'is not valid.' % (handle_type, i)
)
try:
h = Handle(**v)
except TypeError as err:
raise PipelineDescriptionError(
'Arguments for type "%s" of output item #%d '
'in handles description are incorrect:\n%s' %
(handle_type, i, str(err))
)
if not(isinstance(h, handles.OutputHandle) or
isinstance(h, handles.PipeHandle)):
raise PipelineDescriptionError(
'Type "%s" is not a valid output handle.' % handle_type
)
descriptions.append(h)
names.append(v['name'])
if len(set(names)) < len(value):
raise PipelineDescriptionError(
'Names of output items in handles description must be unique.'
)
return descriptions
def _load_pipeline_input(self, site_id):
logger.info('load pipeline inputs')
# Use an in-memory store for pipeline data and only insert outputs
# into the database once the whole pipeline has completed successfully.
store = {
'site_id': site_id,
'pipe': dict(),
'current_figure': list(),
'objects': dict(),
'channels': list()
}
# Load the images, correct them if requested and align them if required.
# NOTE: When the experiment was acquired in "multiplexing" mode,
# images will be automatically aligned, assuming that this is the
# desired behavior.
channel_input = self.project.pipe.description.input.channels
objects_input = self.project.pipe.description.input.objects
with tm.utils.ExperimentSession(self.experiment_id) as session:
site = session.query(tm.Site).get(site_id)
y_offset, x_offset = site.aligned_offset
height = site.aligned_height
width = site.aligned_width
for ch in channel_input:
channel = session.query(
tm.Channel.bit_depth, tm.Channel.id
).\
filter_by(name=ch.name).\
one()
if channel.bit_depth == 16:
dtype = np.uint16
elif channel.bit_depth == 8:
dtype = np.uint8
records = session.query(tm.ChannelImageFile.tpoint).\
filter_by(site_id=site.id,channel_id=channel.id).\
distinct()
tpoints = [r.tpoint for r in records]
n_tpoints = len(tpoints)
records = session.query(tm.ChannelImageFile.zplane).\
filter_by(site_id=site.id,channel_id=channel.id).\
distinct()
zplanes = [r.zplane for r in records]
n_zplanes = len(zplanes)
image_array = np.zeros((height, width, n_zplanes, n_tpoints),
dtype)
if ch.correct:
logger.info(
'load illumination statistics for channel "%s"',
ch.name)
try:
stats_file = session.query(tm.IllumstatsFile).\
join(tm.Channel).\
filter(tm.Channel.name == ch.name).\
one()
except NoResultFound:
raise PipelineDescriptionError(
'No illumination statistics file found for '
'channel "%s"' % ch.name)
stats = stats_file.get()
else:
stats = None
logger.info('load images for channel "%s"', ch.name)
image_files = session.query(tm.ChannelImageFile).\
filter_by(site_id=site.id, channel_id=channel.id).\
all()
for f in image_files:
logger.info('load image %d', f.id)
img = f.get()
if ch.correct:
logger.info('correct image %d', f.id)
img = img.correct(stats)
logger.debug('align image %d', f.id)
img = img.align() # shifted and cropped!
image_array[:, :, f.zplane, f.tpoint] = img.array
store['pipe'][ch.name] = image_array
for obj in objects_input:
mapobject_type = session.query(tm.MapobjectType).\
filter_by(name=obj.name).\
one()
polygons = list()
for t in sorted(tpoints):
zpolys = list()
for z in sorted(zplanes):
zpolys.append(
mapobject_type.get_segmentations_per_site(
site_id=site.id, tpoint=t, zplane=z))
polygons.append(zpolys)
segm_obj = SegmentedObjects(obj.name, obj.name)
segm_obj.add_polygons(polygons, y_offset, x_offset,
(height, width))
store['objects'][segm_obj.name] = segm_obj
store['pipe'][segm_obj.name] = segm_obj.value
# Remove single-dimensions from image arrays.
# NOTE: It would be more consistent to preserve shape, but most people
# will work with 2D/3D images and having to deal with additional
# dimensions would be rather annoying I assume.
for name, img in store['pipe'].iteritems():
store['pipe'][name] = np.squeeze(img)
return store