def mod_primary(composite, mod_id, algorithm): # paths template = composite.templates.get(name='source') # SOURCE TEMPLATE # channel channel, channel_created = composite.channels.get_or_create(name='-primary') # iterate for t in range(composite.series.ts): print(t) # get markers markers = composite.experiment.markers.filter(t=t) primary = np.zeros(composite.series.shape(d=2)) for marker in markers: primary[marker.r-3:marker.r+2, marker.c-3:marker.c+2] = 255 # make blank image and print dots gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=channel, template=template) gon.id_token = generate_id_token('img','Gon') gon.set_origin(0,0,0,t) gon.set_extent(composite.series.rs,composite.series.cs,1) gon.array = primary.copy() gon.save_array(composite.experiment.composite_path, template) gon.save()
def mod_primary(composite, mod_id, algorithm): # paths template = composite.templates.get(name='source') # SOURCE TEMPLATE # channel channel, channel_created = composite.channels.get_or_create( name='-primary') # iterate for t in range(composite.series.ts): print(t) # get markers markers = composite.experiment.markers.filter(t=t) primary = np.zeros(composite.series.shape(d=2)) for marker in markers: primary[marker.r - 3:marker.r + 2, marker.c - 3:marker.c + 2] = 255 # make blank image and print dots gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=channel, template=template) gon.id_token = generate_id_token('img', 'Gon') gon.set_origin(0, 0, 0, t) gon.set_extent(composite.series.rs, composite.series.cs, 1) gon.array = primary.copy() gon.save_array(composite.experiment.composite_path, template) gon.save()
def handle(self, *args, **options): ''' 1. What does this script do? > Use masks to build up larger masks surrounding markers 2. What data structures are input? > Mask, Gon 3. What data structures are output? > Channel, Gon, Mask 4. Is this stage repeated/one-time? > Repeated Steps: 1. load mask gons 2. stack vertically in single array ''' # 1. select composite composite = Composite.objects.get(experiment__name=options['expt'], series__name=options['series']) # 2. Call pmod mod mod = composite.mods.create(id_token=generate_id_token('img', 'Mod'), algorithm='mod_step13_cell_masks') # 3. Run mod print('step13 | processing mod_step13_cell_masks...', end='\r') mod.run() print('step13 | processing mod_step13_cell_masks... done.')
def mod_step09_regions(composite, mod_id, algorithm): # paths template = composite.templates.get(name='region') # REGION TEMPLATE mask_template = composite.templates.get(name='mask') # get region img set that has the region template region_img_set = composite.gons.filter(channel__name='-regionimg', template__name='region') # channel region_channel, region_channel_created = composite.channels.get_or_create(name='-regions') # iterate for t in range(composite.series.ts): region_img = region_img_set.filter(t=t) if region_img.count()==0: region_img = region_img_set.get(t=t-1) else: region_img = region_img_set.get(t=t) # for each image, determine unique values of labelled array # make gon with label array and save region_gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=region_channel, template=template) region_gon.set_origin(0, 0, 0, t) region_gon.set_extent(composite.series.rs, composite.series.cs, 1) # modify image region_array = region_img.load() region_array = region_array[:,:,0] region_array[region_array>0] = 1 region_array, n = label(region_array) region_gon.array = region_array.copy() region_gon.save_array(os.path.join(composite.experiment.mask_path, composite.series.name), template) for unique_value in [u for u in np.unique(region_array) if u>0]: # 1. cut image to single value unique_image = np.zeros(region_array.shape) unique_image[region_array==unique_value] = 1 cut, (r,c,rs,cs) = cut_to_black(unique_image) # 2. make gon with cut image and associate to gon gon = region_gon.gons.create(experiment=composite.experiment, series=composite.series, channel=region_channel, template=mask_template) gon.id_token = generate_id_token('img','Gon') gon.set_origin(r,c,0,t) gon.set_extent(rs,cs,1) gon.array = cut.copy() gon.save_mask(composite.experiment.sub_mask_path) gon.save() # 3. make mask with cut image and associate to gon2 mask = region_gon.masks.create(composite=composite, channel=region_channel, mask_id=unique_value) mask.set_origin(r,c,0) mask.set_extent(rs,cs)
def export_data(self): # composite for datafile composite = self.composites.get() template = composite.templates.get(name='data') id_token = generate_id_token('img', 'DataFile') data_type = 'output' file_name = template.rv.format(self.experiment.name, self.name, id_token, data_type) url = os.path.join(self.experiment.data_path, file_name) # datafile data_file = self.data_files.create(experiment=self.experiment, composite=composite, template=template, id_token=id_token, data_type=data_type, url=url, file_name=file_name) # populate data data_file.data = [cell_instance.line() for cell_instance in self.cell_instances.order_by('cell__pk', 't')] data_file.save_data(headers) data_file.save()
def compose(self): # composite composite, composite_created = self.composites.get_or_create(experiment=self.experiment, id_token=generate_id_token('img', 'Composite')) # templates for template in self.experiment.templates.all(): composite_template, composite_template_created = composite.templates.get_or_create(name=template.name) if composite_template_created: composite_template.rx = template.rx composite_template.rv = template.rv composite_template.save() # iterate over paths for channel in self.experiment.channels.all(): composite_channel, composite_channel_created = composite.channels.get_or_create(name=channel.name) for t in range(self.ts): # path set path_set = self.paths.filter(channel=channel, t=t) # if the total number of paths is less than a great gon, do not make one. Save only individual gons. if path_set.count()==self.zs: # gon gon, gon_created = self.gons.get_or_create(experiment=self.experiment, composite=composite, channel=composite_channel, t=t) if gon_created: gon.set_origin(0,0,0,t) gon.set_extent(self.rs, self.cs, self.zs) for z in range(self.zs): # path path = path_set.get(channel=channel, t=t, z=z) template = composite.templates.get(name=path.template.name) gon.template = template gon.paths.get_or_create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=z) # sub gon sub_gon, sub_gon_created = self.gons.get_or_create(experiment=self.experiment, gon=gon, channel=composite_channel, template=template, t=t, z=z) E = z==self.zs-1 and t==self.ts-1 if sub_gon_created: print('step01 | composing {} series {}... channel {} t{} z{}... created.{}'.format(self.experiment.name, self.name, channel.name, t, z, spacer), end='\n' if E else '\r') sub_gon.set_origin(0,0,z,t) sub_gon.set_extent(self.rs, self.cs, 1) sub_gon.paths.create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=z) else: print('step01 | composing {} series {}... channel {} t{} z{}... already exists.{}'.format(self.experiment.name, self.name, channel.name, t, z, spacer), end='\n' if E else '\r') sub_gon.save() gon.save() else: # disfuse gon structure (reduced, regions) L = len(path_set) - 1 for i, path in enumerate(path_set): E = i==L and t==self.ts-1 print('step01 | composing {} series {}... channel {} t{} z{}... created diffuse.{}'.format(self.experiment.name, self.name, channel.name, t, path.z, spacer), end='\n' if E else '\r') template = composite.templates.get(name=path.template.name) gon, gon_created = self.gons.get_or_create(experiment=self.experiment, composite=composite, channel=composite_channel, template=template, t=t, z=path.z) if gon_created: gon.set_origin(0,0,path.z,t) gon.set_extent(self.rs, self.cs, 1) gon.paths.create(composite=composite, channel=composite_channel, template=template, url=path.url, file_name=path.file_name, t=t, z=path.z) gon.save() composite.save() return composite
def mod_step13_cell_masks(composite, mod_id, algorithm): # paths template = composite.templates.get(name='mask') # MASK TEMPLATE # create batches batch = 0 max_batch_size = 100 # channel cell_mask_channel, cell_mask_channel_created = composite.channels.get_or_create(name='cellmask') # mean # mask_mean_max = np.max([mask.mean for mask in composite.masks.all()]) # iterate over frames for t in range(composite.series.ts): print('step13 | processing mod_step13_cell_masks t{}... '.format(t), end='\r') # one mask for each marker markers = composite.series.markers.filter(t=t) # 1. get masks mask_gon_set = composite.gons.filter(channel__name__in=['pmodreduced','bfreduced'], t=t) bulk = create_bulk_from_image_set(mask_gon_set) for m, marker in enumerate(markers): print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}... '.format(t, m, len(markers)), end='\r') if composite.gons.filter(marker=marker).count()==0: # marker parameters r, c, z = marker.r, marker.c, marker.z other_marker_positions = [(m.r,m.c) for m in markers.exclude(pk=marker.pk)] # get primary mask primary_mask = np.zeros(composite.series.shape(), dtype=float) # blank image mask_uids = [(i, uid) for i,uid in enumerate(bulk.gon_stack[r,c,:]) if uid>0] for i,uid in mask_uids: gon_pk = bulk.rv[i] mask = composite.masks.get(gon__pk=gon_pk, mask_id=uid) mask_array = (bulk.slice(pk=mask.gon.pk)==mask.mask_id).astype(float) # modify mask array based on parameters mask_z, mask_max_z, mask_mean, mask_std = mask.z, mask.max_z, mask.mean, mask.std z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_z)) # suppress z levels at increasing distances from marker max_z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_max_z)) # suppress z levels at increasing distances from marker # mean_term = mask_mean / mask_mean_max # raise mask according to mean std_term = 1.0 mask_array = mask_array * z_term * max_z_term * std_term # add to primary mask primary_mask += mask_array # get secondary mask - get unique masks that touch the edge of the primary mask secondary_mask = np.zeros(composite.series.shape(), dtype=float) # blank image secondary_mask_uids = [] edges = np.where(edge_image(primary_mask>0)) for r, c in zip(*edges): for i,uid in enumerate(bulk.gon_stack[r,c,:]): if (i,uid) not in secondary_mask_uids and (i,uid) not in mask_uids and uid>0: secondary_mask_uids.append((i,uid)) for i,uid in secondary_mask_uids: print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}, secondary {}/{}... '.format(t, m, len(markers), i, len(secondary_mask_uids)), end='\r') gon_pk = bulk.rv[i] mask = composite.masks.get(gon__pk=gon_pk, mask_id=uid) mask_array = (bulk.slice(pk=mask.gon.pk)==mask.mask_id).astype(float) # modify mask array based on parameters mask_z, mask_max_z, mask_mean, mask_std = mask.z, mask.max_z, mask.mean, mask.std z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_z)) # suppress z levels at increasing distances from marker max_z_term = 1.0 / (1.0 + 0.1*np.abs(z - mask_max_z)) # suppress z levels at increasing distances from marker # mean_term = mask_mean / mask_mean_max # raise mask according to mean std_term = 1.0 foreign_marker_condition = 1.0 # if the mask contains a different marker foreign_marker_match = False foreign_marker_counter = 0 while not foreign_marker_match and foreign_marker_counter!=len(other_marker_positions)-1: r, c = other_marker_positions[foreign_marker_counter] foreign_marker_match = (mask_array>0)[r,c] if foreign_marker_match: foreign_marker_condition = 0.0 foreign_marker_counter += 1 mask_array = mask_array * z_term * max_z_term * std_term * foreign_marker_condition # add to primary mask secondary_mask += mask_array print('step13 | processing mod_step13_cell_masks t{}, marker {}/{}, saving square mask... '.format(t, m, len(markers)), end='\n' if t==composite.series.ts-1 else '\r') cell_mask = primary_mask + secondary_mask # finally, mean threshold mask cell_mask[cell_mask<nonzero_mean(cell_mask)] = 0 cell_mask[cell_mask<nonzero_mean(cell_mask)] = 0 # cut to size # I want every mask to be exactly the same size -> 128 pixels wide # I want the centre of the mask to be in the centre of image # Add black space around even past the borders of larger image # 1. determine centre of mass com_r, com_c = com(cell_mask>0) mask_square = np.zeros((256,256), dtype=float) if not np.isnan(com_r): # 2. cut to black and preserve boundaries cut, (cr, cc, crs, ccs) = cut_to_black(cell_mask) # 3. place cut inside square image using the centre of mass and the cut boundaries to hit the centre dr, dc = int(128 + cr - com_r), int(128 + cc - com_c) # 4. preserve coordinates of square to position gon small_r, small_c = mask_square[dr:dr+crs,dc:dc+ccs].shape mask_square[dr:dr+crs,dc:dc+ccs] = cut[:small_r,:small_c] # check batch and make folders, set url if not os.path.exists(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))): os.makedirs(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))) if len(os.listdir(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))))>=max_batch_size: batch += 1 if not os.path.exists(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))): os.makedirs(os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch))) root = os.path.join(composite.experiment.cp2_path, composite.series.name, str(batch)) # CP PATH # cell mask gon cell_mask_gon = composite.gons.create(experiment=composite.experiment, series=composite.series, channel=cell_mask_channel, template=template) cell_mask_gon.set_origin(cr-dr, cc-dc, z, t) cell_mask_gon.set_extent(crs, ccs, 1) id_token = generate_id_token('img','Gon') cell_mask_gon.id_token = id_token file_name = template.rv.format(id_token) url = os.path.join(root, file_name) imsave(url, mask_square.copy()) cell_mask_gon.paths.create(composite=composite, channel=cell_mask_channel, template=template, url=url, file_name=file_name, t=t, z=z) # associate with marker marker.gon = cell_mask_gon cell_mask_gon.marker = marker marker.save() cell_mask_gon.save()