def filter_timepoint_wrap(exp_dir,
filter_excluded=True,
annotation_dir='annotations',
channels=['bf']):
positions = load_data.read_annotations(exp_dir,
annotation_dir=annotation_dir)
if filter_excluded:
positions = load_data.filter_annotations(positions,
load_data.filter_excluded)
def timepoint_filter(position_name, timepoint_name):
return os.path.exists(exp_dir + os.path.sep + position_name +
os.path.sep + timepoint_name + ' comp.png')
def good_pos_filter(position_name, timepoint_name):
if position_name in positions:
return timepoint_filter(position_name, timepoint_name)
else:
return position_name in positions
return load_data.scan_experiment_dir(exp_dir,
channels=channels,
timepoint_filter=good_pos_filter)
def load_masks(experiment_root, mask_root=None):
experiment_root = pathlib.Path(experiment_root)
if mask_root is None:
mask_root = experiment_root / 'derived_data' / 'mask'
mask_root = pathlib.Path(mask_root)
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_subsample_timepoints(experiment_root))
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints)
image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations)
experiment_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter)
# experiment_images_masks = load_data.scan_experiment_dir(experiment_root / 'derived_data' / 'mask', timepoint_filter=image_filter)
for position, position_images in experiment_images.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(mask_root / position / f'{timepoint} bf.png')
# experiment_images = experiment_images_bf.copy()
# for position, position_images in experiment_images.items():
# for timepoint, timepoint_images in position_images.items():
# timepoint_images.append(experiment_images_masks[position][timepoint])
return experiment_images
def filter_latest_images(experiment_root):
annotations = load_data.read_annotations(experiment_root)
good_annotations = load_data.filter_annotations(annotations,
load_data.filter_excluded)
def latelife_filter(position_name, timepoint_name):
return position_name in good_annotations and timepoint_name > good_annotations[
position_name][0]['__last_timepoint_annotated__']
return load_data.scan_experiment_dir(expt_dir,
timepoint_filter=latelife_filter)
def annotate_poses(exp_root, channels='bf'):
ws = pose_annotation.PoseAnnotation(rw,
name='pose',
width_estimator=None,
objective=10,
optocoupler=0.7)
sf = stage_field.StageField(stages=['egg', 'larva', 'adult', 'dead'])
positions = load_data.scan_experiment_dir(exp_root,
channels=channels,
error_on_missing=False)
ea = experiment_annotator.ExperimentAnnotator(rw, exp_root, positions,
[sf, ws])
def filter_good_positions_wrap(experiment_root,
channels='bf',
error_on_missing=False):
positions = load_data.read_annotations(experiment_root)
good_positions = load_data.filter_annotations(
positions, load_data.filter_good_incomplete)
def timepoint_filter(position_name, timepoint_name):
return position_name in good_positions
return load_data.scan_experiment_dir(experiment_root,
timepoint_filter=timepoint_filter,
channels=channels,
error_on_missing=error_on_missing)
def load_derived_images(experiment_root, derived_dir, *additional_filters):
experiment_root = pathlib.Path(experiment_root)
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(
experiment_annotations, load_data.filter_excluded)
for filter in additional_filters:
experiment_annotations = load_data.filter_annotations(
experiment_annotations, filter)
image_filter = elegant_filters.filter_from_elegant_dict(
experiment_annotations)
experiment_images = load_data.scan_experiment_dir(
experiment_root, timepoint_filter=image_filter)
for position, position_images in experiment_images.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(experiment_root / 'derived_data' /
derived_dir / position /
f'{timepoint} bf.png')
return experiment_images
'/mnt/9karray/Sinha_Drew/20180524_spe-9_Run_4/')
timepoint = '2018-05-28t1526' # Early timepoint
timepoint = '2018-06-03t1714' # Late timepoint ~8d ..
if view_files:
try:
rw
except NameError:
rw = ris_widget.RisWidget()
rw.show()
def timepoint_filter(position_name, timepoint_name):
return timepoint_name == timepoint
experiment_images = load_data.scan_experiment_dir(
experiment_dir,
timepoint_filter=timepoint_filter,
channels=['bf', 'green_yellow_excitation_autofluorescence'])
image_names, images = [], []
fl_image_modes = [] # Measurement of the gel
fl_image_center_medians = [
] # Measurement of the food (1100,900) to (1300, 1100)
for position_name, position_images in experiment_images.items():
if position_images[timepoint]:
processed_position_images = []
for image_name in position_images[timepoint]:
raw_image = freeimage.read(str(image_name))
if 'fl' in image_name.name:
fl_image_modes.append(
np.bincount(raw_image.flat)[1:].argmax() + 1)
fl_image_center_medians.append(
combined_image[:image1.shape[0]] = image1
combined_image[image1.shape[0]:] = image2
yield combined_image
# Actual code to pull relevant images
experiment_root = pathlib.Path('/mnt/9karray/Sinha_Drew/20180924_spe-9_fastmovement')
out_dir = pathlib.Path('/home/drew')
worm = '00'
min_age = 12 # hr
max_age = 24
experiment_annotations = load_data.read_annotations(experiment_root)
experiment_annotations = load_data.filter_annotations(experiment_annotations, load_data.filter_excluded)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_adult_timepoints)
experiment_annotations = load_data.filter_annotations(experiment_annotations, elegant_filters.filter_by_age(min_age,max_age,adult_age=True))
image_filter = elegant_filters.filter_from_elegant_dict(experiment_annotations)
# scan_experiment_dir for bf images
print('Scanning...')
bf_images = load_data.scan_experiment_dir(experiment_root, timepoint_filter=image_filter)
bf_generator = write_movie.generate_images_from_files(bf_images['00'])
rep_image = freeimage.read(bf_images['00'][list(bf_images['00'].keys())[0]])
print('Making lab frame mask...')
# scan_experiment_dir for mask images or helper to grab pose masks
position_annotations, timepoint_annotations = experiment_root[worm]
mask_generator = (worm_spline.lab_frame_mask(timepoint_info['bf pose'][0], timepoint_info['bf pose'][1], rep_image.shape) for timepoint_info in timepoint_annotations)
image_generator = write_movie.shrink(yield_rgb(double_image_layout(bf_generator, mask_generator)), factor=4)
write_movie.write_movie(image_generator, out_dir / (f'{worm} {min_age/24:.1f}-{max_age/24:.1f}.replace('.','_')' + '.mp4'),framerate=4)
def process_experiment_with_filter(experiment_root,
model,
image_filter,
mask_root=None,
overwrite_existing=False,
channels='bf',
make_masks=True,
do_annotations=True):
'''
image_filter - filter for scan_experiment_dir
'''
if mask_root is None:
mask_root = pathlib.Path(experiment_root) / 'derived_data' / 'mask'
# Temporary hacks until migration to new elegant complete (while zpl-9000 no longer updates annotations automatically)
process_data.update_annotations(experiment_root)
elegant_hacks.propagate_stages(experiment_root)
start_t = time.time()
positions = load_data.scan_experiment_dir(experiment_root,
timepoint_filter=image_filter,
channels=channels)
scan_t = time.time()
print(f'scanning done after {(scan_t-start_t)} s')
if make_masks:
process = segment_images.segment_positions(positions,
model,
mask_root,
use_gpu=True,
overwrite_existing=False)
if process.stderr:
print(f'Errors during segmentation: {process.stderr}'
) #raise Exception)
#raise Exception()
segment_t = time.time()
print(f'segmenting done after {(segment_t-scan_t)} s')
with (mask_root / 'notes.txt').open('a+') as notes_file:
notes_file.write(
f'{datetime.datetime.today().strftime("%Y-%m-%dt%H%M")} These masks were segmented with model {model}\n'
)
else:
print(f'No segmenting performed')
if do_annotations:
annotations = load_data.read_annotations(experiment_root)
metadata = load_data.read_metadata(experiment_root)
age_factor = metadata.get('age_factor', 1)
width_estimator = worm_widths.WidthEstimator.from_experiment_metadata(
metadata, age_factor)
segment_images.annotate_poses_from_masks(positions, mask_root,
annotations,
overwrite_existing,
width_estimator)
load_data.write_annotations(experiment_root, annotations)
annotation_t = time.time()
print(
f'annotation done after {(annotation_t - segment_t)} s') # ~3.5 hr
else:
print('No annotations done')
################################################################
# User-specified parameters
NAME = "06"
experimental_root = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1'
worm_annotation_files = '/Volumes/9karray/Mosley_Matt/20190408_lin-4_spe-9_20C_pos-1/derived_data/measurements/core_measures/*.tsv'
OUTPUT_DIR = '/Users/zplab/Desktop/DanScripts/MiscFigures/'
# Open worm images and annotation information
rw = ris_widget.RisWidget()
worm_annotations = worm_data.read_worms(worm_annotation_files, name_prefix='')
worm_annotations.sort(
'name') # Optional sort, but easier to follow for debugging
files = load_data.scan_experiment_dir(experimental_root)
# Image annotation data
positions = load_data.read_annotations(experimental_root)
positions = load_data.filter_annotations(positions, load_data.filter_excluded)
# Iterate through all worms, select worm of interest
for worm in worm_annotations:
worm_name = worm.name
# only select the correct worm
if worm_name != NAME:
continue
for i in range(len(worm.td.timepoint)):
# Ensure that worm is an adult (can use other filter depending on dataset)
if worm.td.stage[i] == 'adult':
rw.annotator.close()
del (rw.annotator)
process_data.update_annotations(expt_dir)
experiment_annotations = load_data.read_annotations(
expt_dir, annotation_dir=annotation_dir)
if timepoint_filters:
experiment_annotations = load_data.filter_annotations(
experiment_annotations,
elegant_filters.compose_timepoint_filters(*timepoint_filters))
expt_pos = load_data.scan_experiment_dir(
expt_dir,
channels=channels,
timepoint_filter=lambda position_name, timepoint_name:
not experiment_annotations or
(position_name in experiment_annotations and timepoint_name in
experiment_annotations[position_name][1]))
if show_masks:
mask_root = expt_dir / 'derived_data' / 'mask'
for position, position_images in expt_pos.items():
for timepoint, timepoint_images in position_images.items():
timepoint_images.append(mask_root / position /
f'{timepoint} bf.png')
annotation_fields = []
annotation_fields.append(stage_field.StageField())
if show_poses:
metadata = load_data.read_metadata(expt_dir)
if channels[0] == 'bf':
# Use timepoint filter only for Nicolette's dataset
####
worm_tp_dict = {}
timepoints = worm_annotations.get_time_range('timepoint', min_age=24*2, max_age=24*7)
for i, worm in enumerate(worm_annotations):
tp = timepoints[i]
worm_tp_dict[worm.name] = tp.T[1]
def timepoint_filter(position_name, timepoint_name, tp_dict = worm_tp_dict):
if position_name not in tp_dict.keys():
return False
if timepoint_name in tp_dict[position_name]:
return True
else:
return False
files = load_data.scan_experiment_dir(experimental_root, timepoint_filter=timepoint_filter)
####
# Use this for Matt's dataset
#files = load_data.scan_experiment_dir(experimental_root)
# Image annotation data
positions = load_data.read_annotations(experimental_root)
positions = load_data.filter_annotations(positions, load_data.filter_excluded)
# Location of output files
OUTPUT_DIRECTORY = '/Users/zplab/Desktop/DanScripts/worm_images/---'
# Array with pertinent worm information
worm_image_data = [['name', 'timepoint', 'lifespan', 'hours_remaining']]
# Iterate through worms
for worm in worm_annotations: