def check_stage_annotations(annotations, stages): """Check for incomplete annotations in an experiment Parameters annotations - an OrderedDict of experiment_annotations as returned by load_data.read_annotations stages - A iterable containing the stages that should be annotated (e.g. could be ('larva','adult','dead') for a complete experiment, but only ('larva', 'adult') for an ongoing experiment) Returns annotations OrderedDict for animals with incomplete annotations """ # Create a suitable function to use with filter_positions using a closure def select_by_stage_annotation(position_name, position_annotations, timepoint_annotations): stage_annotations = [ timepoint_annotation.get('stage', '') for timepoint_annotation in timepoint_annotations.values() ] return all([stage in stage_annotations for stage in stages]) good_annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) complete_annotations = load_data.filter_annotations( good_annotations, select_by_stage_annotation ) # Get positions whose stages are not all annotated return complete_annotations
def load_masks(experiment_root, mask_root=None): experiment_root = pathlib.Path(experiment_root) if mask_root is None: mask_root = experiment_root / 'derived_data' / 'mask' mask_root = pathlib.Path(mask_root) experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_subsample_timepoints(experiment_root)) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints) image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations) experiment_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter) # experiment_images_masks = load_data.scan_experiment_dir(experiment_root / 'derived_data' / 'mask', timepoint_filter=image_filter) for position, position_images in experiment_images.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(mask_root / position / f'{timepoint} bf.png') # experiment_images = experiment_images_bf.copy() # for position, position_images in experiment_images.items(): # for timepoint, timepoint_images in position_images.items(): # timepoint_images.append(experiment_images_masks[position][timepoint]) return experiment_images
def run_canonical_measurements(experiment_dir): '''Run standard measurements on the specified experiment directory''' experiment_dir = pathlib.Path(experiment_dir) process_data.update_annotations(experiment_dir) position_features = ['stage_x', 'stage_y', 'starting_stage_z', 'notes'] annotations = load_data.read_annotations(experiment_dir) annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) annotations = load_data.filter_annotations(annotations, filter_worm_positions) #print('warning: Im using a custom filter function') #annotations = load_data.filter_annotations(annotations, lambda name, pa, ta: name < '25') # Remove me later if any([ 'lawn_area' in position_annotations for (position_annotations, timepoint_annotations) in annotations.items() ]): position_features.append('lawn_area') make_basic_measurements(experiment_dir, annotations) process_data.collate_data(experiment_dir, position_features=position_features) return make_pose_measurements(experiment_dir, annotations) process_data.collate_data(experiment_dir, position_features=position_features) #make_mask_measurements(experiment_dir, annotations) image_channels = elegant_hacks.get_image_channels(experiment_dir) print(f'Image channels: {image_channels}') if 'bf_1' in image_channels: print('Found multipass movement channel bf_1; making measurements') make_multipass_measurements(experiment_dir, annotations) process_data.collate_data( experiment_dir, position_features=position_features ) # For convenience since autofluorescence can take a little while.... if 'green_yellow_excitation_autofluorescence' in image_channels or 'autofluorescence' in image_channels: fl_measurement_name = 'autofluorescence' if 'autofluorescence' in image_channels else 'green_yellow_excitation_autofluorescence' print( f'Found autofluorescence channel {fl_measurement_name}; making measurements' ) make_af_measurements(experiment_dir, annotations, fl_measurement_name=fl_measurement_name) process_data.collate_data(experiment_dir, position_features=position_features)
def check_for_kw(expt_dir, kw, filter_good=True, verbose=True): annotations = load_data.read_annotations(expt_dir) if filter_good: annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) kw_annotations = load_data.filter_annotations( annotations, elegant_filters.filter_by_kw(kw)) if verbose: print( f'{len(kw_annotations)}/{len(annotations)} of animals in experiment has kw {kw} {"(minus excluded)" if filter_good else ""}' ) return set(kw_annotations.keys())
def filter_timepoint_wrap(exp_dir, filter_excluded=True, annotation_dir='annotations', channels=['bf']): positions = load_data.read_annotations(exp_dir, annotation_dir=annotation_dir) if filter_excluded: positions = load_data.filter_annotations(positions, load_data.filter_excluded) def timepoint_filter(position_name, timepoint_name): return os.path.exists(exp_dir + os.path.sep + position_name + os.path.sep + timepoint_name + ' comp.png') def good_pos_filter(position_name, timepoint_name): if position_name in positions: return timepoint_filter(position_name, timepoint_name) else: return position_name in positions return load_data.scan_experiment_dir(exp_dir, channels=channels, timepoint_filter=good_pos_filter)
def filter_latest_images(experiment_root): annotations = load_data.read_annotations(experiment_root) good_annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) def latelife_filter(position_name, timepoint_name): return position_name in good_annotations and timepoint_name > good_annotations[ position_name][0]['__last_timepoint_annotated__'] return load_data.scan_experiment_dir(expt_dir, timepoint_filter=latelife_filter)
def check_for_alive(expt_dir): annotations = load_data.read_annotations(expt_dir) good_annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) dead_annotations = check_stage_annotations(good_annotations, ['dead']) print( f'{len(good_annotations)-len(dead_annotations)}/{len(good_annotations)} still alive' ) return set(good_annotations.keys()).difference(set( dead_annotations.keys()))
def load_derived_images(experiment_root, derived_dir, *additional_filters): experiment_root = pathlib.Path(experiment_root) experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations( experiment_annotations, load_data.filter_excluded) for filter in additional_filters: experiment_annotations = load_data.filter_annotations( experiment_annotations, filter) image_filter = elegant_filters.filter_from_elegant_dict( experiment_annotations) experiment_images = load_data.scan_experiment_dir( experiment_root, timepoint_filter=image_filter) for position, position_images in experiment_images.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(experiment_root / 'derived_data' / derived_dir / position / f'{timepoint} bf.png') return experiment_images
def make_gfp_measurements(experiment_root, annotations, fl_measurement_name='gfp', adult_only=True): measures = [process_data.FluorMeasurements(fl_measurement_name)] measurement_name = 'fluorescence_measures' if adult_only: annotations = load_data.filter_annotations( annotations, elegant_filters.filter_by_stage('adult')).copy() process_data.measure_worms(experiment_root, annotations, measures, measurement_name)
def make_mask_measurements(experiment_root, annotations=None, adult_only=True): #process_data.annotate(experiment_root, annotators=[annotate_timepoints]) # Why? experiment_metadata = load_data.read_metadata(experiment_root) microns_per_pixel = 1.3 * 5 / (experiment_metadata['objective'] * experiment_metadata['optocoupler']) measures = [MaskPoseMeasurements(microns_per_pixel=microns_per_pixel)] measurement_name = 'mask_measures' if annotations is None: annotations = load_data.read_annotations(experiment_root) annotations = load_data.filter_annotations(annotations, filter_excluded) annotations = load_data.filter_annotations( annotations, elegant_filters.filter_living_timepoints) if adult_only: annotations = load_data.filter_annotations( annotations, elegant_filters.filter_by_stage('adult')) process_data.measure_worms(experiment_root, annotations, measures, measurement_name)
def get_image_channels(experiment_root): experiment_root = pathlib.Path(experiment_root) annotations = load_data.read_annotations(experiment_root) annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) positions = list(annotations.keys()) image_channels = { image_file.stem.split()[1] for image_file in (experiment_root / positions[0]).iterdir() if image_file.suffix[1:] in ['png', 'tif'] } return image_channels
def make_multipass_measurements(experiment_root, annotations, adult_only=True): experiment_metadata = load_data.read_metadata(experiment_root) microns_per_pixel = 1.3 * 5 / (experiment_metadata['objective'] * experiment_metadata['optocoupler']) measures = [MultipassPoseMeasurements(microns_per_pixel=microns_per_pixel)] measurement_name = 'multipass_measures' if adult_only: annotations = load_data.filter_annotations( annotations, elegant_filters.filter_by_stage('adult')).copy() process_data.measure_worms(experiment_root, annotations, measures, measurement_name)
def enumerate_common_annotations(experiment_dir, bad_kws=None, verbose=True, filter_good=True): if not bad_kws: bad_kws = [['Nw', 'Nh'], ['REFERENCE'], ['CONTAMINATION'], ['DOUBLE WORM', 'TRIPLE WORM'], ['ESCAPE', 'VISITED'], ['LOST'], ['LARVAL', 'DELAYED'], ['FERTILE'], [ 'PVL', 'BURST', ], ['bag\'d'], ['small', 'sickly', 'scrawny', 'mottled']] experiment_dir = pathlib.Path(experiment_dir) inventory_bad_worms = [] annotations = load_data.read_annotations(experiment_dir) if filter_good: original_worms = annotations.keys() annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) excluded_worms = sorted( list(set(original_worms) - set(annotations.keys()))) for kw_group in bad_kws: group_list = [] for kw in kw_group: group_list.extend([ worm for worm, worm_annotations in annotations.items() if kw in worm_annotations[0]['notes'] ]) group_list = utilities.unique_items(group_list) inventory_bad_worms.append(group_list) if verbose: print(f'\n{experiment_dir.name} (n = {len(annotations)})') if filter_good: print(f'(excluded): {len(excluded_worms)} ({excluded_worms})') for kw_group, bad_worms in zip(bad_kws, inventory_bad_worms): print(f'{"/".join(kw_group)}: {len(bad_worms)} ({bad_worms})') utilities.print_table( [[len(bad_worms) for bad_worms in inventory_bad_worms] + [f'{len(annotations)}']], column_names=[f'{"/".join(kw_group)}' for kw_group in bad_kws] + ['Total'], row_names=[experiment_dir.name])
def reset_positions_manual(scope, experiment_dir, *annotation_filters, revert_z=False): '''Reset positions manually for an experiment (i.e. with a separate ris_widget window open) Parameters: scope - ScopeClient object as defined by scope.scope_client experiment_dir - str/pathlib.Path to experiment annotation_filters - Optional variable filters to use to isolate specific positions of interest Call with annotation filters like so: reset_position.reset_positions(scope, experiment_dir, elegant_filters.filter_excluded, elegant_filters.filter_live_animals) ''' experiment_dir = pathlib.Path(experiment_dir) print(f'Traversing {experiment_dir.name}') metadata = load_data.read_metadata(experiment_dir) if annotation_filters: experiment_annotations = load_data.read_annotations(experiment_dir) for filter in annotation_filters: experiment_annotations = load_data.filter_annotations( experiment_annotations, filter) positions = experiment_annotations.keys() else: positions = metadata['positions'].keys() new_positions = poll_positions(scope, metadata, positions, revert_z=revert_z) if new_positions: try: input(f'\nPress any key to save positions; ctrl-c to abort') time_label = time.strftime('%Y%m%d-%H%M-%S') with (experiment_dir / f'experiment_metadata_beforechangingpositions_{time_label}.json' ).open('w') as mdata_file: datafile.json_encode_legible_to_file(metadata, mdata_file) metadata['positions'].update(new_positions) load_data.write_metadata(metadata, experiment_dir) except KeyboardInterrupt: pass else: print('No positions found to reset')
def filter_good_positions_wrap(experiment_root, channels='bf', error_on_missing=False): positions = load_data.read_annotations(experiment_root) good_positions = load_data.filter_annotations( positions, load_data.filter_good_incomplete) def timepoint_filter(position_name, timepoint_name): return position_name in good_positions return load_data.scan_experiment_dir(experiment_root, timepoint_filter=timepoint_filter, channels=channels, error_on_missing=error_on_missing)
def process_experiment(experiment_directory, channels_to_process, num_workers=None, position_filter=None, **process_args): experiment_directory = pathlib.Path(experiment_directory) if num_workers == None: num_workers = multiprocessing.cpu_count() - 2 elif num_workers > multiprocessing.cpu_count(): raise RuntimeError( 'Attempted to run jobs with more workers than cpu\'s!') if position_filter is None: position_filter = load_data.filter_good_complete # Make super_vignette if needed. if not (experiment_directory / 'super_vignette.pickle').exists(): make_super_vignette(experiment_directory) # Make appropriate subdirectories if not (experiment_directory / 'derived_data').exists(): (experiment_directory / 'derived_data').mkdir() if not (experiment_directory / 'derived_data' / 'mask').exists(): (experiment_directory / 'derived_data' / 'mask').mkdir() # Enumerate position directories positions = load_data.read_annotations(experiment_directory) positions = load_data.filter_annotations(positions, position_filter=position_filter) print('Processing the following positions:') print(list(positions.keys()).__repr__()) with multiprocessing.Pool(processes=num_workers) as pool: try: bob = pool.map( functools.partial(process_position_directory, channels_to_process=channels_to_process, **process_args), [experiment_directory / pos for pos in positions]) pool.close() pool.join() print('Terminated successfully') except KeyboardInterrupt: pool.terminate() pool.join() raise
def run_holly_measurements(experiment_dir): position_features = ['stage_x', 'stage_y', 'starting_stage_z', 'notes'] annotations = load_data.read_annotations(experiment_dir) annotations = load_data.filter_annotations(annotations, load_data.filter_excluded) make_basic_measurements(experiment_dir, annotations) make_pose_measurements(experiment_dir, annotations, adult_only=False) process_data.collate_data(experiment_dir, position_features=position_features ) # Make preliminary analysis faster. make_gfp_measurements(experiment_dir, annotations, adult_only=False) make_af_measurements(experiment_dir, annotations, adult_only=False) process_data.collate_data(experiment_dir, position_features=position_features)
def check_for_null_poses(experiment_root, annotation_dir='annotations'): assert pathlib.Path(experiment_root).exists() experiment_annotations = load_data.read_annotations( experiment_root, annotation_dir=annotation_dir) experiment_annotations = load_data.filter_annotations( experiment_annotations, load_data.filter_excluded) poses = ['pose'] for position, position_annotations in experiment_annotations.items(): for timepoint, timepoint_annotations in position_annotations[1].items( ): if 'bf_1 pose' in timepoint_annotations and 'bf_1 pose' not in poses: for i in range(7): poses.append(f'bf_{i+1} pose') for pose_tag in poses: if timepoint_annotations.get( pose_tag, (None, None) )[0] is None and timepoint_annotations['stage'] == 'adult': print( f"Position {position}, timepoint {timepoint} doesn't have a vaild {pose_tag} pose" ) print(f'Checked for poses {poses}')
for image1, image2 in zip(image_generator1, image_generator2): assert (image1.shape == image2.shape).all() combined_image = numpy.zeros((image1.shape[0], 2*image1.shape[1])) combined_image[:image1.shape[0]] = image1 combined_image[image1.shape[0]:] = image2 yield combined_image # Actual code to pull relevant images experiment_root = pathlib.Path('/mnt/9karray/Sinha_Drew/20180924_spe-9_fastmovement') out_dir = pathlib.Path('/home/drew') worm = '00' min_age = 12 # hr max_age = 24 experiment_annotations = load_data.read_annotations(experiment_root) experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints) experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_by_age(min_age,max_age,adult_age=True)) image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations) # scan_experiment_dir for bf images print('Scanning...') bf_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter) bf_generator = write_movie.generate_images_from_files(bf_images['00']) rep_image = freeimage.read(bf_images['00'][list(bf_images['00'].keys())[0]]) print('Making lab frame mask...') # scan_experiment_dir for mask images or helper to grab pose masks position_annotations, timepoint_annotations = experiment_root[worm] mask_generator = (worm_spline.lab_frame_mask(timepoint_info['bf pose'][0], timepoint_info['bf pose'][1], rep_image.shape) for timepoint_info in timepoint_annotations)
def plot_annotation_ls(*annotation_dirs, ax_h=None, calc_adultspan=False, **plot_kws): """Plot survival curves for one or more separate experiments Parameters annotation_dirs - one or more annotation root directories ax_h - optional matplotlib axis objects to plot curves on calc_adultspan - bool flag that toggles whether to calculate lifespan as adultspan; if True, uses the 'adult' timepoint as the starting timepoint of interest; otherwise, uses the 'larva' timepoint plot_kws - optional kw parameters to pass to plt.plot Returns metadata - list of dicts where each entry corresponds to derived metadata about the plotted experiment; this includes: n - number of animals mean - mean lifespan of animals median - median lifespan good_ls - numpy array of good lifespans (fig_h) - matplotlib figure object if supplied ax_h is None (ax_h) - matplotlib axis object if supplied ax_h is None """ if ax_h is None: fig_h, ax_h = plt.subplots() ax_provided = False else: ax_provided = True if type(annotation_dirs[0]) is str: annotation_dirs = [ pathlib.Path(anno_dir) for anno_dir in annotation_dirs ] legend_entries = [] metadata = [] for anno_dir in annotation_dirs: expt_name = anno_dir.name experiment_annotations = load_data.read_annotations( anno_dir.parent, annotation_dir=anno_dir.name) experiment_annotations = load_data.filter_annotations( experiment_annotations, load_data.filter_excluded) lifespans = np.array([]) for position, position_annotations in experiment_annotations.items(): general_annotations, timepoint_annotations = position_annotations timepoints = list(timepoint_annotations.keys()) life_stages = [ timepoint_info.get('stage') for timepoint_info in timepoint_annotations.values() ] if 'dead' in life_stages: if calc_adultspan: birth_timepoint = timepoints[life_stages.index('adult')] else: birth_timepoint = timepoints[life_stages.index('larva')] death_timepoint = timepoints[life_stages.index('dead')] lifespans = np.append( lifespans, (utilities.extract_datetime_fromstr(death_timepoint) - utilities.extract_datetime_fromstr(birth_timepoint) ).total_seconds() / (3600 * 24)) else: # Catch animals that are still alive.... lifespans = np.append(lifespans, np.nan) if any(np.isnan(lifespans)): print(f'Warning! Some worms in {expt_name} still alive.') data_series = plot_spanseries(lifespans, ax_h=ax_h, **plot_kws) expt_metadata = { 'n': len(lifespans), 'mean': np.nanmean(lifespans), 'median': np.nanmedian(lifespans), 'ls': lifespans, } metadata.append(expt_metadata) legend_entries.append(f'{expt_name} (n={len(lifespans)})') ax_h.legend(legend_entries) ax_h.set_ylabel('Proportion Surviving') if calc_adultspan: ax_h.set_xlabel('Days Adulthood') else: ax_h.set_xlabel('Days Post-Hatch') if not ax_provided: return (fig_h, ax_h, metadata) else: return metadata
rw_defined = 'rw' in globals() global rw if not rw_defined: rw = ris_widget.RisWidget() # Allows one to restart annotating in the same ris_widget window if hasattr(rw, 'annotator'): rw.annotator.close() del (rw.annotator) process_data.update_annotations(expt_dir) experiment_annotations = load_data.read_annotations( expt_dir, annotation_dir=annotation_dir) if timepoint_filters: experiment_annotations = load_data.filter_annotations( experiment_annotations, elegant_filters.compose_timepoint_filters(*timepoint_filters)) expt_pos = load_data.scan_experiment_dir( expt_dir, channels=channels, timepoint_filter=lambda position_name, timepoint_name: not experiment_annotations or (position_name in experiment_annotations and timepoint_name in experiment_annotations[position_name][1])) if show_masks: mask_root = expt_dir / 'derived_data' / 'mask' for position, position_images in expt_pos.items(): for timepoint, timepoint_images in position_images.items(): timepoint_images.append(mask_root / position / f'{timepoint} bf.png')
# User-specified parameters NAME = "06" experimental_root = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1' worm_annotation_files = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1/derived_data/measurements/core_measures/*.tsv' OUTPUT_DIR = '/Users/zplab/Desktop/DanScripts/MiscFigures/' # Open worm images and annotation information rw = ris_widget.RisWidget() worm_annotations = worm_data.read_worms(worm_annotation_files, name_prefix='') worm_annotations.sort( 'name') # Optional sort, but easier to follow for debugging files = load_data.scan_experiment_dir(experimental_root) # Image annotation data positions = load_data.read_annotations(experimental_root) positions = load_data.filter_annotations(positions, load_data.filter_excluded) # Iterate through all worms, select worm of interest for worm in worm_annotations: worm_name = worm.name # only select the correct worm if worm_name != NAME: continue for i in range(len(worm.td.timepoint)): # Ensure that worm is an adult (can use other filter depending on dataset) if worm.td.stage[i] == 'adult': image_timepoint = worm.td.timepoint[i] # Convert scope image to worm frame image lab_frame_image = freeimage.read(