def collect_job_output(self, batch):
'''Creates :class:`MapobjectType <tmlib.models.mapobject.MapobjectType>`
instances for :class:`Site <tmlib.models.site.Site>`,
:class:`Well <tmlib.models.well.Well>`,
and :class:`Plate <tmlib.models.plate.Plate>` types and creates for each
object of these classes an instance of
:class:`Mapobject <tmlib.models.mapobject.Mapobject>` and
:class:`MapobjectSegmentation <tmlib.models.mapobject.MapobjectSegmentation>`.
This allows visualizing these objects on the map and using them
for efficient spatial queries.
Parameters
----------
batch: dict
job description
'''
mapobject_mappings = {
'Plates': tm.Plate,
'Wells': tm.Well,
'Sites': tm.Site
}
for name, cls in mapobject_mappings.iteritems():
with tm.utils.ExperimentSession(self.experiment_id,
transaction=False) as session:
logger.info(
'create static mapobject type "%s" for reference type "%s"',
name, cls.__name__)
mapobject_type = session.get_or_create(
tm.MapobjectType,
name=name,
experiment_id=self.experiment_id,
ref_type=cls.__name__)
mapobject_type_id = mapobject_type.id
segmentation_layer = session.get_or_create(
tm.SegmentationLayer, mapobject_type_id=mapobject_type_id)
logger.info('create individual mapobjects of type "%s"', name)
segmentations = dict()
for obj in session.query(cls):
logger.debug('create mapobject for reference object #%d',
obj.id)
if name == 'Sites':
# We need to account for the "multiplexing" edge case.
offset = obj.aligned_offset
image_size = obj.aligned_image_size
else:
offset = obj.offset
image_size = obj.image_size
# First element: x axis
# Second element: inverted (!) y axis
# We further subtract one pixel such that the polygon
# defines the exact boundary of the objects. This is
# crucial for testing whether other objects intersect with
# the border.
ul = (offset[1] + 1, -1 * (offset[0] + 1))
ll = (ul[0], ul[1] - (image_size[0] - 3))
ur = (ul[0] + image_size[1] - 3, ul[1])
lr = (ll[0] + image_size[1] - 3, ll[1])
# Closed circle with coordinates sorted counter-clockwise
contour = np.array([ur, ul, ll, lr, ur])
polygon = shapely.geometry.Polygon(contour)
segmentations[obj.id] = {
'segmentation_layer_id': segmentation_layer.id,
'polygon': polygon
}
logger.debug('delete existing mapobjects of type "%s"', name)
session.query(tm.Mapobject).\
filter_by(mapobject_type_id=mapobject_type_id).\
delete()
logger.debug('add new mapobjects of type "%s"', name)
for key, value in segmentations.iteritems():
mapobject = tm.Mapobject(
partition_key=key, mapobject_type_id=mapobject_type_id)
session.add(mapobject)
session.flush()
logger.debug('add mapobject #%d', mapobject.id)
mapobject_segmentation = tm.MapobjectSegmentation(
partition_key=key,
mapobject_id=mapobject.id,
geom_polygon=value['polygon'],
geom_centroid=value['polygon'].centroid,
segmentation_layer_id=value['segmentation_layer_id'],
)
session.add(mapobject_segmentation)
def add_segmentations(experiment_id, mapobject_type_id):
"""
.. http:post:: /api/experiments/(string:experiment_id)/mapobject_types/(string:mapobject_type_id)/segmentations
Provide segmentations in form of a labeled 2D pixels array
for a given :class:`Site <tmlib.models.site.Site>`.
A :class:`Mapobject <tmlib.models.mapobject.Mapobject>` and
:class:`MapobjectSegmentation <tmlib.models.mapobject.MapobjectSegmentation>`
will be created for each labeled connected pixel component in *image*.
:reqheader Authorization: JWT token issued by the server
:statuscode 200: no error
:statuscode 400: malformed request
:query npz_file: npz file containing the segmentation image "segmentation" (required)
:query plate_name: name of the plate (required)
:query well_name: name of the well (required)
:query well_pos_x: x-coordinate of the site within the well (required)
:query well_pos_y: y-coordinate of the site within the well (required)
:query tpoint: time point (required)
:query zplane: z-plane (required)
"""
data = request.get_json()
plate_name = data.get('plate_name')
well_name = data.get('well_name')
well_pos_x = int(data.get('well_pos_x'))
well_pos_y = int(data.get('well_pos_y'))
zplane = int(data.get('zplane'))
tpoint = int(data.get('tpoint'))
align = is_true(request.args.get('align')) # TODO
logger.info(
'add segmentations for mapobject type %d of experiment %d at '
'plate "%s", well "%s", well position %d/%d, zplane %d, time point %d',
mapobject_type_id, experiment_id, plate_name, well_name, well_pos_y,
well_pos_x, zplane, tpoint)
npz_file = base64.b64decode(data.get('npz_file'))
pixels = np.load(BytesIO(npz_file))["segmentation"]
array = np.array(pixels, dtype=np.int32)
labels = np.unique(array[array > 0])
n_objects = len(labels)
with tm.utils.ExperimentSession(experiment_id) as session:
segmentation_layer = session.get_or_create(
tm.SegmentationLayer,
mapobject_type_id=mapobject_type_id,
tpoint=tpoint,
zplane=zplane)
segmentation_layer_id = segmentation_layer.id
site = session.query(tm.Site).\
join(tm.Well).\
join(tm.Plate).\
filter(
tm.Plate.name == plate_name, tm.Well.name == well_name,
tm.Site.y == well_pos_y, tm.Site.x == well_pos_x
).\
one()
if align:
y_offset, x_offset = site.aligned_offset
if array.shape != site.aligned_image_size:
raise MalformedRequestError('Image has wrong dimensions')
else:
y_offset, x_offset = site.offset
if array.shape != site.image_size:
raise MalformedRequestError('Image has wrong dimensions')
site_id = site.id
metadata = SegmentationImageMetadata(mapobject_type_id, site_id,
tpoint, zplane)
image = SegmentationImage(array, metadata)
existing_segmentations_map = dict(
session.query(
tm.MapobjectSegmentation.label,
tm.MapobjectSegmentation.mapobject_id
).\
join(tm.Mapobject).\
filter(
tm.MapobjectSegmentation.label.in_(labels.tolist()),
tm.Mapobject.mapobject_type_id == mapobject_type_id,
tm.MapobjectSegmentation.partition_key == site_id
).\
all()
)
with tm.utils.ExperimentSession(experiment_id, False) as session:
segmentations = list()
for label, polygon in image.extract_polygons(y_offset, x_offset):
mapobject = tm.Mapobject(site_id, mapobject_type_id)
if label in existing_segmentations_map:
# A parent mapobject with the same label may already exist,
# because it got already created for another zplane/tpoint.
# The segmentation for the given zplane/tpoint must not yet
# exist, however. This will lead to an error upon insertion.
mapobject.id = existing_segmentations_map[label]
else:
session.add(mapobject)
session.flush()
s = tm.MapobjectSegmentation(
partition_key=site_id,
mapobject_id=mapobject.id,
geom_polygon=polygon,
geom_centroid=polygon.centroid,
segmentation_layer_id=segmentation_layer_id,
label=label)
segmentations.append(s)
session.bulk_ingest(segmentations)
return jsonify(message='ok')
def _save_pipeline_outputs(self, store, assume_clean_state):
logger.info('save pipeline outputs')
objects_output = self.project.pipe.description.output.objects
for item in objects_output:
as_polygons = item.as_polygons
store['objects'][item.name].save = True
store['objects'][item.name].represent_as_polygons = as_polygons
with tm.utils.ExperimentSession(self.experiment_id, False) as session:
layer = session.query(tm.ChannelLayer).first()
mapobject_type_ids = dict()
segmentation_layer_ids = dict()
objects_to_save = dict()
feature_ids = collections.defaultdict(dict)
for obj_name, segm_objs in store['objects'].iteritems():
if segm_objs.save:
logger.info('objects of type "%s" are saved', obj_name)
objects_to_save[obj_name] = segm_objs
else:
logger.info('objects of type "%s" are not saved', obj_name)
continue
logger.debug('add object type "%s"', obj_name)
mapobject_type = session.get_or_create(
tm.MapobjectType,
experiment_id=self.experiment_id,
name=obj_name,
ref_type=tm.Site.__name__)
mapobject_type_ids[obj_name] = mapobject_type.id
# Create a feature values entry for each segmented object at
# each time point.
logger.info('add features for objects of type "%s"', obj_name)
for feature_name in segm_objs.measurements[0].columns:
logger.debug('add feature "%s"', feature_name)
feature = session.get_or_create(
tm.Feature,
name=feature_name,
mapobject_type_id=mapobject_type_ids[obj_name],
is_aggregate=False)
feature_ids[obj_name][feature_name] = feature.id
for (t, z), plane in segm_objs.iter_planes():
segmentation_layer = session.get_or_create(
tm.SegmentationLayer,
mapobject_type_id=mapobject_type.id,
tpoint=t,
zplane=z)
segmentation_layer_ids[(obj_name, t, z)] = \
segmentation_layer.id
site = session.query(tm.Site).get(store['site_id'])
y_offset, x_offset = site.aligned_offset
mapobject_ids = dict()
for obj_name, segm_objs in objects_to_save.iteritems():
if not assume_clean_state:
# Delete existing mapobjects for this site, which were
# generated in a previous run of the same pipeline. In case
# they were passed as inputs don't delete them.
inputs = self.project.pipe.description.input.objects
if obj_name not in inputs:
logger.info(
'delete segmentations for existing mapobjects of '
'type "%s"', obj_name)
session.query(tm.Mapobject).\
filter_by(
mapobject_type_id=mapobject_type_ids[obj_name],
partition_key=store['site_id']
).\
delete()
# Create a mapobject for each segmented object, i.e. each
# pixel component having a unique label.
logger.info('add objects of type "%s"', obj_name)
# TODO: Can we avoid these multiple loops?
# Is the bottleneck inserting objects into the db or Python?
mapobjects = [
tm.Mapobject(
partition_key=store['site_id'],
mapobject_type_id=mapobject_type_ids[obj_name])
for _ in segm_objs.labels
]
logger.info('insert objects into database')
# FIXME: does this update the id attribute?
session.bulk_ingest(mapobjects)
session.flush()
mapobject_ids = {
label: mapobjects[i].id
for i, label in enumerate(segm_objs.labels)
}
# Create a polygon and/or point for each segmented object
# based on the cooridinates of their contours and centroids,
# respectively.
logger.info('add segmentations for objects of type "%s"',
obj_name)
mapobject_segmentations = list()
if segm_objs.represent_as_polygons:
logger.debug('represent segmented objects as polygons')
iterator = segm_objs.iter_polygons(y_offset, x_offset)
for t, z, label, polygon in iterator:
logger.debug(
'add segmentation for object #%d at '
'tpoint %d and zplane %d', label, t, z)
if polygon.is_empty:
logger.warn(
'object #%d of type %s doesn\'t have a polygon',
label, obj_name)
# TODO: Shall we rather raise an Exception here???
# At the moment we remove the corresponding
# mapobjects in the collect phase.
continue
mapobject_segmentations.append(
tm.MapobjectSegmentation(
partition_key=store['site_id'],
label=label,
geom_polygon=polygon,
geom_centroid=polygon.centroid,
mapobject_id=mapobject_ids[label],
segmentation_layer_id=segmentation_layer_ids[(
obj_name, t, z)],
))
else:
logger.debug('represent segmented objects only as points')
iterator = segm_objs.iter_points(y_offset, x_offset)
for t, z, label, centroid in iterator:
logger.debug(
'add segmentation for object #%d at '
'tpoint %d and zplane %d', label, t, z)
mapobject_segmentations.append(
tm.MapobjectSegmentation(
partition_key=store['site_id'],
label=label,
geom_polygon=None,
geom_centroid=centroid,
mapobject_id=mapobject_ids[label],
segmentation_layer_id=segmentation_layer_ids[(
obj_name, t, z)]))
logger.info('insert segmentations into database')
session.bulk_ingest(mapobject_segmentations)
logger.info('add feature values for objects of type "%s"',
obj_name)
logger.debug('round feature values to 6 decimals')
feature_values = list()
for t, data in enumerate(segm_objs.measurements):
data = data.round(6) # single!
if data.empty:
logger.warn('empty measurement at time point %d', t)
continue
elif data.shape[0] < len(mapobject_ids):
# We clean up these objects in the collect phase.
logger.error('missing feature values')
elif data.shape[0] > len(mapobject_ids):
# Not sure this could happen.
logger.error('too many feature values')
column_lut = feature_ids[obj_name]
for label, c in data.rename(columns=column_lut).iterrows():
logger.debug(
'add values for mapobject #%d at time point %d',
label, t)
values = dict(
zip(c.index.astype(str), c.values.astype(str)))
feature_values.append(
tm.FeatureValues(partition_key=store['site_id'],
mapobject_id=mapobject_ids[label],
tpoint=t,
values=values))
logger.debug('insert feature values into db table')
session.bulk_ingest(feature_values)