def mod_primary(composite, mod_id, algorithm):
# paths
template = composite.templates.get(name='source') # SOURCE TEMPLATE
# channel
channel, channel_created = composite.channels.get_or_create(name='-primary')
# iterate
for t in range(composite.series.ts):
print(t)
# get markers
markers = composite.experiment.markers.filter(t=t)
primary = np.zeros(composite.series.shape(d=2))
for marker in markers:
primary[marker.r-3:marker.r+2, marker.c-3:marker.c+2] = 255
# make blank image and print dots
gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=channel, template=template)
gon.id_token = generate_id_token('img','Gon')
gon.set_origin(0,0,0,t)
gon.set_extent(composite.series.rs,composite.series.cs,1)
gon.array = primary.copy()
gon.save_array(composite.experiment.composite_path, template)
gon.save()
def mod_primary(composite, mod_id, algorithm):
# paths
template = composite.templates.get(name='source') # SOURCE TEMPLATE
# channel
channel, channel_created = composite.channels.get_or_create(
name='-primary')
# iterate
for t in range(composite.series.ts):
print(t)
# get markers
markers = composite.experiment.markers.filter(t=t)
primary = np.zeros(composite.series.shape(d=2))
for marker in markers:
primary[marker.r - 3:marker.r + 2, marker.c - 3:marker.c + 2] = 255
# make blank image and print dots
gon = composite.gons.create(experiment=composite.experiment,
series=composite.series,
channel=channel,
template=template)
gon.id_token = generate_id_token('img', 'Gon')
gon.set_origin(0, 0, 0, t)
gon.set_extent(composite.series.rs, composite.series.cs, 1)
gon.array = primary.copy()
gon.save_array(composite.experiment.composite_path, template)
gon.save()
def handle(self, *args, **options):
'''
1. What does this script do?
> Use masks to build up larger masks surrounding markers
2. What data structures are input?
> Mask, Gon
3. What data structures are output?
> Channel, Gon, Mask
4. Is this stage repeated/one-time?
> Repeated
Steps:
1. load mask gons
2. stack vertically in single array
'''
# 1. select composite
composite = Composite.objects.get(experiment__name=options['expt'],
series__name=options['series'])
# 2. Call pmod mod
mod = composite.mods.create(id_token=generate_id_token('img', 'Mod'),
algorithm='mod_step13_cell_masks')
# 3. Run mod
print('step13 | processing mod_step13_cell_masks...', end='\r')
mod.run()
print('step13 | processing mod_step13_cell_masks... done.')
def mod_step09_regions(composite, mod_id, algorithm):
# paths
template = composite.templates.get(name='region') # REGION TEMPLATE
mask_template = composite.templates.get(name='mask')
# get region img set that has the region template
region_img_set = composite.gons.filter(channel__name='-regionimg', template__name='region')
# channel
region_channel, region_channel_created = composite.channels.get_or_create(name='-regions')
# iterate
for t in range(composite.series.ts):
region_img = region_img_set.filter(t=t)
if region_img.count()==0:
region_img = region_img_set.get(t=t-1)
else:
region_img = region_img_set.get(t=t)
# for each image, determine unique values of labelled array
# make gon with label array and save
region_gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=region_channel, template=template)
region_gon.set_origin(0, 0, 0, t)
region_gon.set_extent(composite.series.rs, composite.series.cs, 1)
# modify image
region_array = region_img.load()
region_array = region_array[:,:,0]
region_array[region_array>0] = 1
region_array, n = label(region_array)
region_gon.array = region_array.copy()
region_gon.save_array(os.path.join(composite.experiment.mask_path, composite.series.name), template)
for unique_value in [u for u in np.unique(region_array) if u>0]:
# 1. cut image to single value
unique_image = np.zeros(region_array.shape)
unique_image[region_array==unique_value] = 1
cut, (r,c,rs,cs) = cut_to_black(unique_image)
# 2. make gon with cut image and associate to gon
gon = region_gon.gons.create(experiment=composite.experiment, series=composite.series, channel=region_channel, template=mask_template)
gon.id_token = generate_id_token('img','Gon')
gon.set_origin(r,c,0,t)
gon.set_extent(rs,cs,1)
gon.array = cut.copy()
gon.save_mask(composite.experiment.sub_mask_path)
gon.save()
# 3. make mask with cut image and associate to gon2
mask = region_gon.masks.create(composite=composite, channel=region_channel, mask_id=unique_value)
mask.set_origin(r,c,0)
mask.set_extent(rs,cs)
def export_data(self):
# composite for datafile
composite = self.composites.get()
template = composite.templates.get(name='data')
id_token = generate_id_token('img', 'DataFile')
data_type = 'output'
file_name = template.rv.format(self.experiment.name, self.name, id_token, data_type)
url = os.path.join(self.experiment.data_path, file_name)
# datafile
data_file = self.data_files.create(experiment=self.experiment, composite=composite, template=template, id_token=id_token, data_type=data_type, url=url, file_name=file_name)
# populate data
data_file.data = [cell_instance.line() for cell_instance in self.cell_instances.order_by('cell__pk', 't')]
data_file.save_data(headers)
data_file.save()
def compose(self):
# composite
composite, composite_created = self.composites.get_or_create(experiment=self.experiment, id_token=generate_id_token('img', 'Composite'))
# templates
for template in self.experiment.templates.all():
composite_template, composite_template_created = composite.templates.get_or_create(name=template.name)
if composite_template_created:
composite_template.rx = template.rx
composite_template.rv = template.rv
composite_template.save()
# iterate over paths
for channel in self.experiment.channels.all():
composite_channel, composite_channel_created = composite.channels.get_or_create(name=channel.name)
for t in range(self.ts):
# path set
path_set = self.paths.filter(channel=channel, t=t)
# if the total number of paths is less than a great gon, do not make one. Save only individual gons.
if path_set.count()==self.zs:
# gon
gon, gon_created = self.gons.get_or_create(experiment=self.experiment, composite=composite, channel=composite_channel, t=t)
if gon_created:
gon.set_origin(0,0,0,t)
gon.set_extent(self.rs, self.cs, self.zs)
for z in range(self.zs):
# path
path = path_set.get(channel=channel, t=t, z=z)
template = composite.templates.get(name=path.template.name)
gon.template = template
gon.paths.get_or_create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=z)
# sub gon
sub_gon, sub_gon_created = self.gons.get_or_create(experiment=self.experiment, gon=gon, channel=composite_channel, template=template, t=t, z=z)
E = z==self.zs-1 and t==self.ts-1
if sub_gon_created:
print('step01 | composing {} series {}... channel {} t{} z{}... created.{}'.format(self.experiment.name, self.name, channel.name, t, z, spacer), end='\n' if E else '\r')
sub_gon.set_origin(0,0,z,t)
sub_gon.set_extent(self.rs, self.cs, 1)
sub_gon.paths.create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=z)
else:
print('step01 | composing {} series {}... channel {} t{} z{}... already exists.{}'.format(self.experiment.name, self.name, channel.name, t, z, spacer), end='\n' if E else '\r')
sub_gon.save()
gon.save()
else: # disfuse gon structure (reduced, regions)
L = len(path_set) - 1
for i, path in enumerate(path_set):
E = i==L and t==self.ts-1
print('step01 | composing {} series {}... channel {} t{} z{}... created diffuse.{}'.format(self.experiment.name, self.name, channel.name, t, path.z, spacer), end='\n' if E else '\r')
template = composite.templates.get(name=path.template.name)
gon, gon_created = self.gons.get_or_create(experiment=self.experiment, composite=composite, channel=composite_channel, template=template, t=t, z=path.z)
if gon_created:
gon.set_origin(0,0,path.z,t)
gon.set_extent(self.rs, self.cs, 1)
gon.paths.create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=path.z)
gon.save()
composite.save()
return composite
def mod_step13_cell_masks(composite, mod_id, algorithm):
# paths
template = composite.templates.get(name='mask') # MASK TEMPLATE
# create batches
batch = 0
max_batch_size = 100
# channel
cell_mask_channel, cell_mask_channel_created = composite.channels.get_or_create(name='cellmask')
# mean
# mask_mean_max = np.max([mask.mean for mask in composite.masks.all()])
# iterate over frames
for t in range(composite.series.ts):
print('step13 | processing mod_step13_cell_masks t{}... '.format(t), end='\r')
# one mask for each marker
markers = composite.series.markers.filter(t=t)
# 1. get masks
mask_gon_set = composite.gons.filter(channel__name__in=['pmodreduced','bfreduced'], t=t)
bulk = create_bulk_from_image_set(mask_gon_set)
for m, marker in enumerate(markers):
print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}... '.format(t, m, len(markers)), end='\r')
if composite.gons.filter(marker=marker).count()==0:
# marker parameters
r, c, z = marker.r, marker.c, marker.z
other_marker_positions = [(m.r,m.c) for m in markers.exclude(pk=marker.pk)]
# get primary mask
primary_mask = np.zeros(composite.series.shape(), dtype=float) # blank image
mask_uids = [(i, uid) for i,uid in enumerate(bulk.gon_stack[r,c,:]) if uid>0]
for i,uid in mask_uids:
gon_pk = bulk.rv[i]
mask = composite.masks.get(gon__pk=gon_pk, mask_id=uid)
mask_array = (bulk.slice(pk=mask.gon.pk)==mask.mask_id).astype(float)
# modify mask array based on parameters
mask_z, mask_max_z, mask_mean, mask_std = mask.z, mask.max_z, mask.mean, mask.std
z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_z)) # suppress z levels at increasing distances from marker
max_z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_max_z)) # suppress z levels at increasing distances from marker
# mean_term = mask_mean / mask_mean_max # raise mask according to mean
std_term = 1.0
mask_array = mask_array * z_term * max_z_term * std_term
# add to primary mask
primary_mask += mask_array
# get secondary mask - get unique masks that touch the edge of the primary mask
secondary_mask = np.zeros(composite.series.shape(), dtype=float) # blank image
secondary_mask_uids = []
edges = np.where(edge_image(primary_mask>0))
for r, c in zip(*edges):
for i,uid in enumerate(bulk.gon_stack[r,c,:]):
if (i,uid) not in secondary_mask_uids and (i,uid) not in mask_uids and uid>0:
secondary_mask_uids.append((i,uid))
for i,uid in secondary_mask_uids:
print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}, secondary {}/{}... '.format(t, m, len(markers), i, len(secondary_mask_uids)), end='\r')
gon_pk = bulk.rv[i]
mask = composite.masks.get(gon__pk=gon_pk, mask_id=uid)
mask_array = (bulk.slice(pk=mask.gon.pk)==mask.mask_id).astype(float)
# modify mask array based on parameters
mask_z, mask_max_z, mask_mean, mask_std = mask.z, mask.max_z, mask.mean, mask.std
z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_z)) # suppress z levels at increasing distances from marker
max_z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_max_z)) # suppress z levels at increasing distances from marker
# mean_term = mask_mean / mask_mean_max # raise mask according to mean
std_term = 1.0
foreign_marker_condition = 1.0 # if the mask contains a different marker
foreign_marker_match = False
foreign_marker_counter = 0
while not foreign_marker_match and foreign_marker_counter!=len(other_marker_positions)-1:
r, c = other_marker_positions[foreign_marker_counter]
foreign_marker_match = (mask_array>0)[r,c]
if foreign_marker_match:
foreign_marker_condition = 0.0
foreign_marker_counter += 1
mask_array = mask_array * z_term * max_z_term * std_term * foreign_marker_condition
# add to primary mask
secondary_mask += mask_array
print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}, saving square mask... '.format(t, m, len(markers)), end='\n' if t==composite.series.ts-1 else '\r')
cell_mask = primary_mask + secondary_mask
# finally, mean threshold mask
cell_mask[cell_mask<nonzero_mean(cell_mask)] = 0
cell_mask[cell_mask<nonzero_mean(cell_mask)] = 0
# cut to size
# I want every mask to be exactly the same size -> 128 pixels wide
# I want the centre of the mask to be in the centre of image
# Add black space around even past the borders of larger image
# 1. determine centre of mass
com_r, com_c = com(cell_mask>0)
mask_square = np.zeros((256,256), dtype=float)
if not np.isnan(com_r):
# 2. cut to black and preserve boundaries
cut, (cr, cc, crs, ccs) = cut_to_black(cell_mask)
# 3. place cut inside square image using the centre of mass and the cut boundaries to hit the centre
dr, dc = int(128 + cr - com_r), int(128 + cc - com_c)
# 4. preserve coordinates of square to position gon
small_r, small_c = mask_square[dr:dr+crs,dc:dc+ccs].shape
mask_square[dr:dr+crs,dc:dc+ccs] = cut[:small_r,:small_c]
# check batch and make folders, set url
if not os.path.exists(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))):
os.makedirs(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch)))
if len(os.listdir(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))))>=max_batch_size:
batch += 1
if not os.path.exists(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))):
os.makedirs(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch)))
root = os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch)) # CP PATH
# cell mask gon
cell_mask_gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=cell_mask_channel, template=template)
cell_mask_gon.set_origin(cr-dr, cc-dc, z, t)
cell_mask_gon.set_extent(crs, ccs, 1)
id_token = generate_id_token('img','Gon')
cell_mask_gon.id_token = id_token
file_name = template.rv.format(id_token)
url = os.path.join(root, file_name)
imsave(url, mask_square.copy())
cell_mask_gon.paths.create(composite=composite, channel=cell_mask_channel, template=template, url=url, file_name=file_name, t=t, z=z)
# associate with marker
marker.gon = cell_mask_gon
cell_mask_gon.marker = marker
marker.save()
cell_mask_gon.save()
