def filter_timepoint_wrap(exp_dir, filter_excluded=True, annotation_dir='annotations', channels=['bf']): positions = load_data.read_annotations(exp_dir, annotation_dir=annotation_dir) if filter_excluded: positions = load_data.filter_annotations(positions, load_data.filter_excluded) def timepoint_filter(position_name, timepoint_name): return os.path.exists(exp_dir + os.path.sep + position_name + os.path.sep + timepoint_name + ' comp.png') def good_pos_filter(position_name, timepoint_name): if position_name in positions: return timepoint_filter(position_name, timepoint_name) else: return position_name in positions return load_data.scan_experiment_dir(exp_dir, channels=channels, timepoint_filter=good_pos_filter)
def load_masks(experiment_root, mask_root=None): experiment_root = pathlib.Path(experiment_root) if mask_root is None: mask_root = experiment_root / 'derived_data' / 'mask' mask_root = pathlib.Path(mask_root) experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_subsample_timepoints(experiment_root)) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints) image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations) experiment_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter) # experiment_images_masks = load_data.scan_experiment_dir(experiment_root / 'derived_data' / 'mask', timepoint_filter=image_filter) for position, position_images in experiment_images.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(mask_root / position / f'{timepoint} bf.png') # experiment_images = experiment_images_bf.copy() # for position, position_images in experiment_images.items(): # for timepoint, timepoint_images in position_images.items(): # timepoint_images.append(experiment_images_masks[position][timepoint]) return experiment_images
def filter_latest_images(experiment_root): annotations = load_data.read_annotations(experiment_root) good_annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) def latelife_filter(position_name, timepoint_name): return position_name in good_annotations and timepoint_name > good_annotations[ position_name][0]['__last_timepoint_annotated__'] return load_data.scan_experiment_dir(expt_dir, timepoint_filter=latelife_filter)
def annotate_poses(exp_root, channels='bf'): ws = pose_annotation.PoseAnnotation(rw, name='pose', width_estimator=None, objective=10, optocoupler=0.7) sf = stage_field.StageField(stages=['egg', 'larva', 'adult', 'dead']) positions = load_data.scan_experiment_dir(exp_root, channels=channels, error_on_missing=False) ea = experiment_annotator.ExperimentAnnotator(rw, exp_root, positions, [sf, ws])
def filter_good_positions_wrap(experiment_root, channels='bf', error_on_missing=False): positions = load_data.read_annotations(experiment_root) good_positions = load_data.filter_annotations( positions, load_data.filter_good_incomplete) def timepoint_filter(position_name, timepoint_name): return position_name in good_positions return load_data.scan_experiment_dir(experiment_root, timepoint_filter=timepoint_filter, channels=channels, error_on_missing=error_on_missing)
def load_derived_images(experiment_root, derived_dir, *additional_filters): experiment_root = pathlib.Path(experiment_root) experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations( experiment_annotations, load_data.filter_excluded) for filter in additional_filters: experiment_annotations = load_data.filter_annotations( experiment_annotations, filter) image_filter = elegant_filters.filter_from_elegant_dict( experiment_annotations) experiment_images = load_data.scan_experiment_dir( experiment_root, timepoint_filter=image_filter) for position, position_images in experiment_images.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(experiment_root / 'derived_data' / derived_dir / position / f'{timepoint} bf.png') return experiment_images
'/mnt/9karray/Sinha_Drew/20180524_spe-9_Run_4/') timepoint = '2018-05-28t1526' # Early timepoint timepoint = '2018-06-03t1714' # Late timepoint ~8d .. if view_files: try: rw except NameError: rw = ris_widget.RisWidget() rw.show() def timepoint_filter(position_name, timepoint_name): return timepoint_name == timepoint experiment_images = load_data.scan_experiment_dir( experiment_dir, timepoint_filter=timepoint_filter, channels=['bf', 'green_yellow_excitation_autofluorescence']) image_names, images = [], [] fl_image_modes = [] # Measurement of the gel fl_image_center_medians = [ ] # Measurement of the food (1100,900) to (1300, 1100) for position_name, position_images in experiment_images.items(): if position_images[timepoint]: processed_position_images = [] for image_name in position_images[timepoint]: raw_image = freeimage.read(str(image_name)) if 'fl' in image_name.name: fl_image_modes.append( np.bincount(raw_image.flat)[1:].argmax() + 1) fl_image_center_medians.append(
combined_image[:image1.shape[0]] = image1 combined_image[image1.shape[0]:] = image2 yield combined_image # Actual code to pull relevant images experiment_root = pathlib.Path('/mnt/9karray/Sinha_Drew/20180924_spe-9_fastmovement') out_dir = pathlib.Path('/home/drew') worm = '00' min_age = 12 # hr max_age = 24 experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_by_age(min_age,max_age,adult_age=True)) image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations) # scan_experiment_dir for bf images print('Scanning...') bf_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter) bf_generator = write_movie.generate_images_from_files(bf_images['00']) rep_image = freeimage.read(bf_images['00'][list(bf_images['00'].keys())[0]]) print('Making lab frame mask...') # scan_experiment_dir for mask images or helper to grab pose masks position_annotations, timepoint_annotations = experiment_root[worm] mask_generator = (worm_spline.lab_frame_mask(timepoint_info['bf pose'][0], timepoint_info['bf pose'][1], rep_image.shape) for timepoint_info in timepoint_annotations) image_generator = write_movie.shrink(yield_rgb(double_image_layout(bf_generator, mask_generator)), factor=4) write_movie.write_movie(image_generator, out_dir / (f'{worm} {min_age/24:.1f}-{max_age/24:.1f}.replace('.','_')' + '.mp4'),framerate=4)
def process_experiment_with_filter(experiment_root, model, image_filter, mask_root=None, overwrite_existing=False, channels='bf', make_masks=True, do_annotations=True): ''' image_filter - filter for scan_experiment_dir ''' if mask_root is None: mask_root = pathlib.Path(experiment_root) / 'derived_data' / 'mask' # Temporary hacks until migration to new elegant complete (while zpl-9000 no longer updates annotations automatically) process_data.update_annotations(experiment_root) elegant_hacks.propagate_stages(experiment_root) start_t = time.time() positions = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter, channels=channels) scan_t = time.time() print(f'scanning done after {(scan_t-start_t)} s') if make_masks: process = segment_images.segment_positions(positions, model, mask_root, use_gpu=True, overwrite_existing=False) if process.stderr: print(f'Errors during segmentation: {process.stderr}' ) #raise Exception) #raise Exception() segment_t = time.time() print(f'segmenting done after {(segment_t-scan_t)} s') with (mask_root / 'notes.txt').open('a+') as notes_file: notes_file.write( f'{datetime.datetime.today().strftime("%Y-%m-%dt%H%M")} These masks were segmented with model {model}\n' ) else: print(f'No segmenting performed') if do_annotations: annotations = load_data.read_annotations(experiment_root) metadata = load_data.read_metadata(experiment_root) age_factor = metadata.get('age_factor', 1) width_estimator = worm_widths.WidthEstimator.from_experiment_metadata( metadata, age_factor) segment_images.annotate_poses_from_masks(positions, mask_root, annotations, overwrite_existing, width_estimator) load_data.write_annotations(experiment_root, annotations) annotation_t = time.time() print( f'annotation done after {(annotation_t - segment_t)} s') # ~3.5 hr else: print('No annotations done')
################################################################ # User-specified parameters NAME = "06" experimental_root = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1' worm_annotation_files = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1/derived_data/measurements/core_measures/*.tsv' OUTPUT_DIR = '/Users/zplab/Desktop/DanScripts/MiscFigures/' # Open worm images and annotation information rw = ris_widget.RisWidget() worm_annotations = worm_data.read_worms(worm_annotation_files, name_prefix='') worm_annotations.sort( 'name') # Optional sort, but easier to follow for debugging files = load_data.scan_experiment_dir(experimental_root) # Image annotation data positions = load_data.read_annotations(experimental_root) positions = load_data.filter_annotations(positions, load_data.filter_excluded) # Iterate through all worms, select worm of interest for worm in worm_annotations: worm_name = worm.name # only select the correct worm if worm_name != NAME: continue for i in range(len(worm.td.timepoint)): # Ensure that worm is an adult (can use other filter depending on dataset) if worm.td.stage[i] == 'adult':
rw.annotator.close() del (rw.annotator) process_data.update_annotations(expt_dir) experiment_annotations = load_data.read_annotations( expt_dir, annotation_dir=annotation_dir) if timepoint_filters: experiment_annotations = load_data.filter_annotations( experiment_annotations, elegant_filters.compose_timepoint_filters(*timepoint_filters)) expt_pos = load_data.scan_experiment_dir( expt_dir, channels=channels, timepoint_filter=lambda position_name, timepoint_name: not experiment_annotations or (position_name in experiment_annotations and timepoint_name in experiment_annotations[position_name][1])) if show_masks: mask_root = expt_dir / 'derived_data' / 'mask' for position, position_images in expt_pos.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(mask_root / position / f'{timepoint} bf.png') annotation_fields = [] annotation_fields.append(stage_field.StageField()) if show_poses: metadata = load_data.read_metadata(expt_dir) if channels[0] == 'bf':
# Use timepoint filter only for Nicolette's dataset #### worm_tp_dict = {} timepoints = worm_annotations.get_time_range('timepoint', min_age=24*2, max_age=24*7) for i, worm in enumerate(worm_annotations): tp = timepoints[i] worm_tp_dict[worm.name] = tp.T[1] def timepoint_filter(position_name, timepoint_name, tp_dict = worm_tp_dict): if position_name not in tp_dict.keys(): return False if timepoint_name in tp_dict[position_name]: return True else: return False files = load_data.scan_experiment_dir(experimental_root, timepoint_filter=timepoint_filter) #### # Use this for Matt's dataset #files = load_data.scan_experiment_dir(experimental_root) # Image annotation data positions = load_data.read_annotations(experimental_root) positions = load_data.filter_annotations(positions, load_data.filter_excluded) # Location of output files OUTPUT_DIRECTORY = '/Users/zplab/Desktop/DanScripts/worm_images/---' # Array with pertinent worm information worm_image_data = [['name', 'timepoint', 'lifespan', 'hours_remaining']] # Iterate through worms for worm in worm_annotations: