def check_stage_annotations(annotations, stages):
"""Check for incomplete annotations in an experiment
Parameters
annotations - an OrderedDict of experiment_annotations as returned
by load_data.read_annotations
stages - A iterable containing the stages that should be annotated
(e.g. could be ('larva','adult','dead') for a complete experiment,
but only ('larva', 'adult') for an ongoing experiment)
Returns
annotations OrderedDict for animals with incomplete annotations
"""
# Create a suitable function to use with filter_positions using a closure
def select_by_stage_annotation(position_name, position_annotations,
timepoint_annotations):
stage_annotations = [
timepoint_annotation.get('stage', '')
for timepoint_annotation in timepoint_annotations.values()
]
return all([stage in stage_annotations for stage in stages])
good_annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
complete_annotations = load_data.filter_annotations(
good_annotations, select_by_stage_annotation
) # Get positions whose stages are not all annotated
return complete_annotations
def load_masks(experiment_root, mask_root=None):
experiment_root = pathlib.Path(experiment_root)
if mask_root is None:
mask_root = experiment_root / 'derived_data' / 'mask'
mask_root = pathlib.Path(mask_root)
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_subsample_timepoints(experiment_root))
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints)
image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations)
experiment_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter)
# experiment_images_masks = load_data.scan_experiment_dir(experiment_root / 'derived_data' / 'mask', timepoint_filter=image_filter)
for position, position_images in experiment_images.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(mask_root / position / f'{timepoint} bf.png')
# experiment_images = experiment_images_bf.copy()
# for position, position_images in experiment_images.items():
# for timepoint, timepoint_images in position_images.items():
# timepoint_images.append(experiment_images_masks[position][timepoint])
return experiment_images
def run_canonical_measurements(experiment_dir):
'''Run standard measurements on the specified experiment directory'''
experiment_dir = pathlib.Path(experiment_dir)
process_data.update_annotations(experiment_dir)
position_features = ['stage_x', 'stage_y', 'starting_stage_z', 'notes']
annotations = load_data.read_annotations(experiment_dir)
annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
annotations = load_data.filter_annotations(annotations,
filter_worm_positions)
#print('warning: Im using a custom filter function')
#annotations = load_data.filter_annotations(annotations, lambda name, pa, ta: name < '25') # Remove me later
if any([
'lawn_area' in position_annotations
for (position_annotations,
timepoint_annotations) in annotations.items()
]):
position_features.append('lawn_area')
make_basic_measurements(experiment_dir, annotations)
process_data.collate_data(experiment_dir,
position_features=position_features)
return
make_pose_measurements(experiment_dir, annotations)
process_data.collate_data(experiment_dir,
position_features=position_features)
#make_mask_measurements(experiment_dir, annotations)
image_channels = elegant_hacks.get_image_channels(experiment_dir)
print(f'Image channels: {image_channels}')
if 'bf_1' in image_channels:
print('Found multipass movement channel bf_1; making measurements')
make_multipass_measurements(experiment_dir, annotations)
process_data.collate_data(
experiment_dir, position_features=position_features
) # For convenience since autofluorescence can take a little while....
if 'green_yellow_excitation_autofluorescence' in image_channels or 'autofluorescence' in image_channels:
fl_measurement_name = 'autofluorescence' if 'autofluorescence' in image_channels else 'green_yellow_excitation_autofluorescence'
print(
f'Found autofluorescence channel {fl_measurement_name}; making measurements'
)
make_af_measurements(experiment_dir,
annotations,
fl_measurement_name=fl_measurement_name)
process_data.collate_data(experiment_dir,
position_features=position_features)
def check_for_kw(expt_dir, kw, filter_good=True, verbose=True):
annotations = load_data.read_annotations(expt_dir)
if filter_good:
annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
kw_annotations = load_data.filter_annotations(
annotations, elegant_filters.filter_by_kw(kw))
if verbose:
print(
f'{len(kw_annotations)}/{len(annotations)} of animals in experiment has kw {kw} {"(minus excluded)" if filter_good else ""}'
)
return set(kw_annotations.keys())
def filter_timepoint_wrap(exp_dir,
filter_excluded=True,
annotation_dir='annotations',
channels=['bf']):
positions = load_data.read_annotations(exp_dir,
annotation_dir=annotation_dir)
if filter_excluded:
positions = load_data.filter_annotations(positions,
load_data.filter_excluded)
def timepoint_filter(position_name, timepoint_name):
return os.path.exists(exp_dir + os.path.sep + position_name +
os.path.sep + timepoint_name + ' comp.png')
def good_pos_filter(position_name, timepoint_name):
if position_name in positions:
return timepoint_filter(position_name, timepoint_name)
else:
return position_name in positions
return load_data.scan_experiment_dir(exp_dir,
channels=channels,
timepoint_filter=good_pos_filter)
def filter_latest_images(experiment_root):
annotations = load_data.read_annotations(experiment_root)
good_annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
def latelife_filter(position_name, timepoint_name):
return position_name in good_annotations and timepoint_name > good_annotations[
position_name][0]['__last_timepoint_annotated__']
return load_data.scan_experiment_dir(expt_dir,
timepoint_filter=latelife_filter)
def check_for_alive(expt_dir):
annotations = load_data.read_annotations(expt_dir)
good_annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
dead_annotations = check_stage_annotations(good_annotations, ['dead'])
print(
f'{len(good_annotations)-len(dead_annotations)}/{len(good_annotations)} still alive'
)
return set(good_annotations.keys()).difference(set(
dead_annotations.keys()))
def load_derived_images(experiment_root, derived_dir, *additional_filters):
experiment_root = pathlib.Path(experiment_root)
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(
experiment_annotations, load_data.filter_excluded)
for filter in additional_filters:
experiment_annotations = load_data.filter_annotations(
experiment_annotations, filter)
image_filter = elegant_filters.filter_from_elegant_dict(
experiment_annotations)
experiment_images = load_data.scan_experiment_dir(
experiment_root, timepoint_filter=image_filter)
for position, position_images in experiment_images.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(experiment_root / 'derived_data' /
derived_dir / position /
f'{timepoint} bf.png')
return experiment_images
def make_gfp_measurements(experiment_root,
annotations,
fl_measurement_name='gfp',
adult_only=True):
measures = [process_data.FluorMeasurements(fl_measurement_name)]
measurement_name = 'fluorescence_measures'
if adult_only:
annotations = load_data.filter_annotations(
annotations, elegant_filters.filter_by_stage('adult')).copy()
process_data.measure_worms(experiment_root, annotations, measures,
measurement_name)
def make_mask_measurements(experiment_root, annotations=None, adult_only=True):
#process_data.annotate(experiment_root, annotators=[annotate_timepoints]) # Why?
experiment_metadata = load_data.read_metadata(experiment_root)
microns_per_pixel = 1.3 * 5 / (experiment_metadata['objective'] *
experiment_metadata['optocoupler'])
measures = [MaskPoseMeasurements(microns_per_pixel=microns_per_pixel)]
measurement_name = 'mask_measures'
if annotations is None:
annotations = load_data.read_annotations(experiment_root)
annotations = load_data.filter_annotations(annotations,
filter_excluded)
annotations = load_data.filter_annotations(
annotations, elegant_filters.filter_living_timepoints)
if adult_only:
annotations = load_data.filter_annotations(
annotations, elegant_filters.filter_by_stage('adult'))
process_data.measure_worms(experiment_root, annotations, measures,
measurement_name)
def get_image_channels(experiment_root):
experiment_root = pathlib.Path(experiment_root)
annotations = load_data.read_annotations(experiment_root)
annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
positions = list(annotations.keys())
image_channels = {
image_file.stem.split()[1]
for image_file in (experiment_root / positions[0]).iterdir()
if image_file.suffix[1:] in ['png', 'tif']
}
return image_channels
def make_multipass_measurements(experiment_root, annotations, adult_only=True):
experiment_metadata = load_data.read_metadata(experiment_root)
microns_per_pixel = 1.3 * 5 / (experiment_metadata['objective'] *
experiment_metadata['optocoupler'])
measures = [MultipassPoseMeasurements(microns_per_pixel=microns_per_pixel)]
measurement_name = 'multipass_measures'
if adult_only:
annotations = load_data.filter_annotations(
annotations, elegant_filters.filter_by_stage('adult')).copy()
process_data.measure_worms(experiment_root, annotations, measures,
measurement_name)
def enumerate_common_annotations(experiment_dir,
bad_kws=None,
verbose=True,
filter_good=True):
if not bad_kws:
bad_kws = [['Nw', 'Nh'], ['REFERENCE'], ['CONTAMINATION'],
['DOUBLE WORM', 'TRIPLE WORM'], ['ESCAPE', 'VISITED'],
['LOST'], ['LARVAL', 'DELAYED'], ['FERTILE'],
[
'PVL',
'BURST',
], ['bag\'d'], ['small', 'sickly', 'scrawny', 'mottled']]
experiment_dir = pathlib.Path(experiment_dir)
inventory_bad_worms = []
annotations = load_data.read_annotations(experiment_dir)
if filter_good:
original_worms = annotations.keys()
annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
excluded_worms = sorted(
list(set(original_worms) - set(annotations.keys())))
for kw_group in bad_kws:
group_list = []
for kw in kw_group:
group_list.extend([
worm for worm, worm_annotations in annotations.items()
if kw in worm_annotations[0]['notes']
])
group_list = utilities.unique_items(group_list)
inventory_bad_worms.append(group_list)
if verbose:
print(f'\n{experiment_dir.name} (n = {len(annotations)})')
if filter_good:
print(f'(excluded): {len(excluded_worms)} ({excluded_worms})')
for kw_group, bad_worms in zip(bad_kws, inventory_bad_worms):
print(f'{"/".join(kw_group)}: {len(bad_worms)} ({bad_worms})')
utilities.print_table(
[[len(bad_worms)
for bad_worms in inventory_bad_worms] + [f'{len(annotations)}']],
column_names=[f'{"/".join(kw_group)}'
for kw_group in bad_kws] + ['Total'],
row_names=[experiment_dir.name])
def reset_positions_manual(scope,
experiment_dir,
*annotation_filters,
revert_z=False):
'''Reset positions manually for an experiment (i.e. with a separate ris_widget window open)
Parameters:
scope - ScopeClient object as defined by scope.scope_client
experiment_dir - str/pathlib.Path to experiment
annotation_filters - Optional variable filters to use to isolate specific positions of interest
Call with annotation filters like so:
reset_position.reset_positions(scope, experiment_dir, elegant_filters.filter_excluded, elegant_filters.filter_live_animals)
'''
experiment_dir = pathlib.Path(experiment_dir)
print(f'Traversing {experiment_dir.name}')
metadata = load_data.read_metadata(experiment_dir)
if annotation_filters:
experiment_annotations = load_data.read_annotations(experiment_dir)
for filter in annotation_filters:
experiment_annotations = load_data.filter_annotations(
experiment_annotations, filter)
positions = experiment_annotations.keys()
else:
positions = metadata['positions'].keys()
new_positions = poll_positions(scope,
metadata,
positions,
revert_z=revert_z)
if new_positions:
try:
input(f'\nPress any key to save positions; ctrl-c to abort')
time_label = time.strftime('%Y%m%d-%H%M-%S')
with (experiment_dir /
f'experiment_metadata_beforechangingpositions_{time_label}.json'
).open('w') as mdata_file:
datafile.json_encode_legible_to_file(metadata, mdata_file)
metadata['positions'].update(new_positions)
load_data.write_metadata(metadata, experiment_dir)
except KeyboardInterrupt:
pass
else:
print('No positions found to reset')
def filter_good_positions_wrap(experiment_root,
channels='bf',
error_on_missing=False):
positions = load_data.read_annotations(experiment_root)
good_positions = load_data.filter_annotations(
positions, load_data.filter_good_incomplete)
def timepoint_filter(position_name, timepoint_name):
return position_name in good_positions
return load_data.scan_experiment_dir(experiment_root,
timepoint_filter=timepoint_filter,
channels=channels,
error_on_missing=error_on_missing)
def process_experiment(experiment_directory,
channels_to_process,
num_workers=None,
position_filter=None,
**process_args):
experiment_directory = pathlib.Path(experiment_directory)
if num_workers == None:
num_workers = multiprocessing.cpu_count() - 2
elif num_workers > multiprocessing.cpu_count():
raise RuntimeError(
'Attempted to run jobs with more workers than cpu\'s!')
if position_filter is None:
position_filter = load_data.filter_good_complete
# Make super_vignette if needed.
if not (experiment_directory / 'super_vignette.pickle').exists():
make_super_vignette(experiment_directory)
# Make appropriate subdirectories
if not (experiment_directory / 'derived_data').exists():
(experiment_directory / 'derived_data').mkdir()
if not (experiment_directory / 'derived_data' / 'mask').exists():
(experiment_directory / 'derived_data' / 'mask').mkdir()
# Enumerate position directories
positions = load_data.read_annotations(experiment_directory)
positions = load_data.filter_annotations(positions,
position_filter=position_filter)
print('Processing the following positions:')
print(list(positions.keys()).__repr__())
with multiprocessing.Pool(processes=num_workers) as pool:
try:
bob = pool.map(
functools.partial(process_position_directory,
channels_to_process=channels_to_process,
**process_args),
[experiment_directory / pos for pos in positions])
pool.close()
pool.join()
print('Terminated successfully')
except KeyboardInterrupt:
pool.terminate()
pool.join()
raise
def run_holly_measurements(experiment_dir):
position_features = ['stage_x', 'stage_y', 'starting_stage_z', 'notes']
annotations = load_data.read_annotations(experiment_dir)
annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
make_basic_measurements(experiment_dir, annotations)
make_pose_measurements(experiment_dir, annotations, adult_only=False)
process_data.collate_data(experiment_dir,
position_features=position_features
) # Make preliminary analysis faster.
make_gfp_measurements(experiment_dir, annotations, adult_only=False)
make_af_measurements(experiment_dir, annotations, adult_only=False)
process_data.collate_data(experiment_dir,
position_features=position_features)
def check_for_null_poses(experiment_root, annotation_dir='annotations'):
assert pathlib.Path(experiment_root).exists()
experiment_annotations = load_data.read_annotations(
experiment_root, annotation_dir=annotation_dir)
experiment_annotations = load_data.filter_annotations(
experiment_annotations, load_data.filter_excluded)
poses = ['pose']
for position, position_annotations in experiment_annotations.items():
for timepoint, timepoint_annotations in position_annotations[1].items(
):
if 'bf_1 pose' in timepoint_annotations and 'bf_1 pose' not in poses:
for i in range(7):
poses.append(f'bf_{i+1} pose')
for pose_tag in poses:
if timepoint_annotations.get(
pose_tag, (None, None)
)[0] is None and timepoint_annotations['stage'] == 'adult':
print(
f"Position {position}, timepoint {timepoint} doesn't have a vaild {pose_tag} pose"
)
print(f'Checked for poses {poses}')
for image1, image2 in zip(image_generator1, image_generator2):
assert (image1.shape == image2.shape).all()
combined_image = numpy.zeros((image1.shape[0], 2*image1.shape[1]))
combined_image[:image1.shape[0]] = image1
combined_image[image1.shape[0]:] = image2
yield combined_image
# Actual code to pull relevant images
experiment_root = pathlib.Path('/mnt/9karray/Sinha_Drew/20180924_spe-9_fastmovement')
out_dir = pathlib.Path('/home/drew')
worm = '00'
min_age = 12 # hr
max_age = 24
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_by_age(min_age,max_age,adult_age=True))
image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations)
# scan_experiment_dir for bf images
print('Scanning...')
bf_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter)
bf_generator = write_movie.generate_images_from_files(bf_images['00'])
rep_image = freeimage.read(bf_images['00'][list(bf_images['00'].keys())[0]])
print('Making lab frame mask...')
# scan_experiment_dir for mask images or helper to grab pose masks
position_annotations, timepoint_annotations = experiment_root[worm]
mask_generator = (worm_spline.lab_frame_mask(timepoint_info['bf pose'][0], timepoint_info['bf pose'][1], rep_image.shape) for timepoint_info in timepoint_annotations)
def plot_annotation_ls(*annotation_dirs,
ax_h=None,
calc_adultspan=False,
**plot_kws):
"""Plot survival curves for one or more separate experiments
Parameters
annotation_dirs - one or more annotation root directories
ax_h - optional matplotlib axis objects to plot curves on
calc_adultspan - bool flag that toggles whether to calculate lifespan as adultspan;
if True, uses the 'adult' timepoint as the starting timepoint of interest;
otherwise, uses the 'larva' timepoint
plot_kws - optional kw parameters to pass to plt.plot
Returns
metadata - list of dicts where each entry corresponds to
derived metadata about the plotted experiment; this
includes:
n - number of animals
mean - mean lifespan of animals
median - median lifespan
good_ls - numpy array of good lifespans
(fig_h) - matplotlib figure object if supplied ax_h is None
(ax_h) - matplotlib axis object if supplied ax_h is None
"""
if ax_h is None:
fig_h, ax_h = plt.subplots()
ax_provided = False
else:
ax_provided = True
if type(annotation_dirs[0]) is str:
annotation_dirs = [
pathlib.Path(anno_dir) for anno_dir in annotation_dirs
]
legend_entries = []
metadata = []
for anno_dir in annotation_dirs:
expt_name = anno_dir.name
experiment_annotations = load_data.read_annotations(
anno_dir.parent, annotation_dir=anno_dir.name)
experiment_annotations = load_data.filter_annotations(
experiment_annotations, load_data.filter_excluded)
lifespans = np.array([])
for position, position_annotations in experiment_annotations.items():
general_annotations, timepoint_annotations = position_annotations
timepoints = list(timepoint_annotations.keys())
life_stages = [
timepoint_info.get('stage')
for timepoint_info in timepoint_annotations.values()
]
if 'dead' in life_stages:
if calc_adultspan:
birth_timepoint = timepoints[life_stages.index('adult')]
else:
birth_timepoint = timepoints[life_stages.index('larva')]
death_timepoint = timepoints[life_stages.index('dead')]
lifespans = np.append(
lifespans,
(utilities.extract_datetime_fromstr(death_timepoint) -
utilities.extract_datetime_fromstr(birth_timepoint)
).total_seconds() / (3600 * 24))
else: # Catch animals that are still alive....
lifespans = np.append(lifespans, np.nan)
if any(np.isnan(lifespans)):
print(f'Warning! Some worms in {expt_name} still alive.')
data_series = plot_spanseries(lifespans, ax_h=ax_h, **plot_kws)
expt_metadata = {
'n': len(lifespans),
'mean': np.nanmean(lifespans),
'median': np.nanmedian(lifespans),
'ls': lifespans,
}
metadata.append(expt_metadata)
legend_entries.append(f'{expt_name} (n={len(lifespans)})')
ax_h.legend(legend_entries)
ax_h.set_ylabel('Proportion Surviving')
if calc_adultspan:
ax_h.set_xlabel('Days Adulthood')
else:
ax_h.set_xlabel('Days Post-Hatch')
if not ax_provided:
return (fig_h, ax_h, metadata)
else:
return metadata
rw_defined = 'rw' in globals()
global rw
if not rw_defined: rw = ris_widget.RisWidget()
# Allows one to restart annotating in the same ris_widget window
if hasattr(rw, 'annotator'):
rw.annotator.close()
del (rw.annotator)
process_data.update_annotations(expt_dir)
experiment_annotations = load_data.read_annotations(
expt_dir, annotation_dir=annotation_dir)
if timepoint_filters:
experiment_annotations = load_data.filter_annotations(
experiment_annotations,
elegant_filters.compose_timepoint_filters(*timepoint_filters))
expt_pos = load_data.scan_experiment_dir(
expt_dir,
channels=channels,
timepoint_filter=lambda position_name, timepoint_name:
not experiment_annotations or
(position_name in experiment_annotations and timepoint_name in
experiment_annotations[position_name][1]))
if show_masks:
mask_root = expt_dir / 'derived_data' / 'mask'
for position, position_images in expt_pos.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(mask_root / position /
f'{timepoint} bf.png')
# User-specified parameters
NAME = "06"
experimental_root = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1'
worm_annotation_files = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1/derived_data/measurements/core_measures/*.tsv'
OUTPUT_DIR = '/Users/zplab/Desktop/DanScripts/MiscFigures/'
# Open worm images and annotation information
rw = ris_widget.RisWidget()
worm_annotations = worm_data.read_worms(worm_annotation_files, name_prefix='')
worm_annotations.sort(
'name') # Optional sort, but easier to follow for debugging
files = load_data.scan_experiment_dir(experimental_root)
# Image annotation data
positions = load_data.read_annotations(experimental_root)
positions = load_data.filter_annotations(positions, load_data.filter_excluded)
# Iterate through all worms, select worm of interest
for worm in worm_annotations:
worm_name = worm.name
# only select the correct worm
if worm_name != NAME:
continue
for i in range(len(worm.td.timepoint)):
# Ensure that worm is an adult (can use other filter depending on dataset)
if worm.td.stage[i] == 'adult':
image_timepoint = worm.td.timepoint[i]
# Convert scope image to worm frame image
lab_frame_image = freeimage.read(
