def update_z_positions(data_dir, scope):
"""Interactively update the z positions for an existing experiment.
New positions are written to a 'z_updates' metadata dictionary, which is
used by the acquisiton script to override the previous focal position.
Parameters:
data_dir: experiment directory with existing experiment_metadata.json
file.
scope: scope object.
"""
data_dir = pathlib.Path(data_dir)
experiment_metadata_path = data_dir / 'experiment_metadata.json'
with experiment_metadata_path.open() as f:
experiment_metadata = json.load(f)
positions = experiment_metadata['positions']
new_z = {}
for position_name, (x,y,z) in sorted(positions.items()):
position_dir = data_dir / position_name
position_metadata_path = position_dir / 'position_metadata.json'
with position_metadata_path.open() as f:
position_metadata = json.load(f)
for m in position_metadata[::-1]:
if 'fine_z' in m:
z = m['fine_z']
break
scope.stage.position = x, y, z
input('refine position {} (press enter when done)'.format(position_name))
new_z[position_name] = scope.stage.z
experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z
datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def prune_z(experiment_dir):
'''
For when I needed to rescue some scripts after Zach's update to new code
'''
experiment_dir = pathlib.Path(experiment_dir)
for position_dir in sorted(list(experiment_dir.iterdir())):
if position_dir.is_dir() and position_dir.parts[-1][0].isnumeric():
shutil.copyfile(str(position_dir/'position_metadata.json'), str(position_dir/'position_metadata_old.json'))
position_metadata = json.load((position_dir/'position_metadata.json').open('r'))
if type(position_metadata[-1]['fine_z']) is list:
position_metadata[-1]['fine_z'] = position_metadata[-1]['fine_z'][0]
datafile.json_encode_atomic_legible_to_file(position_metadata, (position_dir/'position_metadata.json'))
def write_metadata(self):
"""Write the current set of metadata for all timepoints back to the metadata file."""
if self._metadata is None:
self._load_metadata()
# read from self._metadata instead of gathering from all timepoints in self
# in case some timepoints have been filtered just temporarily -- don't want to
# blow away their metadata
metadata_list = [
dict(timepoint=name, **rest)
for name, rest in sorted(self._metadata.items())
]
datafile.json_encode_atomic_legible_to_file(metadata_list,
self.metadata_file)
def update_z_positions(data_dir, scope):
"""Interactively update the z positions for an existing experiment.
New positions are written to a 'z_updates' metadata dictionary, which is
used by the acquisiton script to override the previous focal position.
Parameters:
data_dir: experiment directory with existing experiment_metadata.json
file.
scope: scope object.
"""
data_dir = pathlib.Path(data_dir)
experiment_metadata_path = data_dir / 'experiment_metadata.json'
with experiment_metadata_path.open() as f:
experiment_metadata = json.load(f)
positions = experiment_metadata['positions']
new_z = {}
for position_name, (x, y, z) in sorted(positions.items()):
position_dir = data_dir / position_name
position_metadata_path = position_dir / 'position_metadata.json'
with position_metadata_path.open() as f:
position_metadata = json.load(f)
for m in position_metadata[::-1]:
if 'fine_z' in m:
z = m['fine_z']
break
scope.stage.position = x, y, z
input(
'refine position {} (press enter when done)'.format(position_name))
new_z[position_name] = scope.stage.z
experiment_metadata.setdefault(
'z_updates', {})[datetime.datetime.now().isoformat()] = new_z
datafile.json_encode_atomic_legible_to_file(experiment_metadata,
experiment_metadata_path)
def update_z_positions(data_dir, scope):
"""Interactively update the z positions for an existing experiment.
New positions are written to a 'z_updates' metadata dictionary, which is
used by the acquisiton script to override the previous focal position.
Parameters:
data_dir: experiment directory with existing experiment_metadata.json
file.
scope: scope object.
"""
data_dir = pathlib.Path(data_dir)
experiment_metadata_path, experiment_metadata = metadata_util.get_experiment_metadata(data_dir)
positions = experiment_metadata['positions']
new_z = {}
for position_name in sorted(positions.keys()):
position_dir, metadata_path, position_metadata = metadata_util.get_position_metadata(data_dir position_name)
scope.stage.position = metadata_util.get_latest_coordinates(position_name, experiment_metadata, position_metadata)
input('refine position {} (press enter when done)'.format(position_name))
new_z[position_name] = scope.stage.z
experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z
datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def _write_atomic_json(self, out_path, data):
datafile.json_encode_atomic_legible_to_file(data, out_path)
def update_positions(data_dir,
scope,
image_type='bf',
dry_run=False,
ignore_autofocus_z=False):
"""Interactively update the xyz positions for an existing experiment.
New z positions are written to a 'z_updates' metadata dictionary, which is
used by the acquisiton script to override the previous focal position. New
x,y positions are written directly to the experiment metadata. (The old
positions are stored in the metadata file as "old_positions".)
Parameters:
data_dir: experiment directory with existing experiment_metadata.json
file.
scope: scope object
dry_run: if True, don't actually write files
ignore_autofocus_z: if True, don't use the last-autofocused z value;
instead just use the value from the original metadata file.
"""
data_dir = pathlib.Path(data_dir)
experiment_metadata_path = data_dir / 'experiment_metadata.json'
with experiment_metadata_path.open() as f:
experiment_metadata = json.load(f)
positions = experiment_metadata['positions']
new_positions = {}
names, xyzs = zip(*positions.items())
old_xys = []
new_xys = []
rotation = numpy.eye(2)
translation = [0, 0]
position_order = _farthest_points(numpy.array(xyzs)[:, :2])
viewer = scope_viewer_widget.ScopeViewerWidget(scope)
viewer.show()
viewer.layers.append(layer.Layer())
viewer.layers[0].tint = 0, 1, 0 # green
viewer.layers[1].tint = 1, 0, 1 # purple
with scope.camera.in_state(live_mode=True), scope.tl.lamp.in_state(
enabled=True), scope.stage.in_state(async_=True):
for i in position_order:
position_name = names[i]
x, y, z = xyzs[i]
# now apply current guess of rotation and translation to get estimated
# new position
x, y = numpy.dot([x, y], rotation) + translation
scope.stage.position = x, y, z
position_dir = data_dir / position_name
position_metadata_path = position_dir / 'position_metadata.json'
with position_metadata_path.open() as f:
position_metadata = json.load(f)
if not ignore_autofocus_z:
for m in position_metadata[::-1]:
if 'fine_z' in m:
z = m['fine_z']
break
image = None
for m in position_metadata[::-1]:
timepoint = m['timepoint']
image_f = position_dir / f'{timepoint} {image_type}.png'
if image_f.exists():
image = freeimage.read(image_f)
break
scope.stage.wait()
if image is None:
print(
f'Could not find any {image_type} image for position {position_name}; skipping'
)
continue
if len(viewer.live_image_page) == 2:
viewer.live_image_page[1] = image
else:
viewer.live_image_page.append(image)
out = viewer.input(
'refine position {} (press enter when done, or press "x" if the position is no longer available)'
.format(position_name))
if out.strip().lower() == 'x':
continue
old_xys.append(x, y)
x, y, z = scope.stage.position
new_positions[position_name] = x, y, z
new_xys.append(x, y)
rotation, _, translation, _ = fit_affine.fit_affine(
old_xys, new_xys, find_scale=False, find_translation=True)
angle = numpy.arctan2(rotation[0, 1], rotation[0,
0]) * 180 / numpy.pi
micron_trans = translation * 1000
print(f'\t estimated rotation angle: {angle:.1f}°')
print(
f'\t estimated translation: ({micron_trans[0]:.0f}, {micron_trans[1]:.0f}) µm'
)
new_z = {
position_name: z
for position_name, (x, y, z) in new_positions.items()
}
experiment_metadata.setdefault(
'z_updates', {})[datetime.datetime.now().isoformat()] = new_z
experiment_metadata['old_positions'] = positions
experiment_metadata['positions'] = new_positions
if not dry_run:
datafile.json_encode_atomic_legible_to_file(experiment_metadata,
experiment_metadata_path)
def update_positions(data_dir, scope, image_type='bf', dry_run=False, ignore_autofocus_z=False):
"""Interactively update the xyz positions for an existing experiment.
New z positions are written to a 'z_updates' metadata dictionary, which is
used by the acquisiton script to override the previous focal position. New
x,y positions are written directly to the experiment metadata. (The old
positions are stored in the metadata file as "old_positions".)
Parameters:
data_dir: experiment directory with existing experiment_metadata.json
file.
scope: scope object
dry_run: if True, don't actually write files
ignore_autofocus_z: if True, don't use the last-autofocused z value;
instead just use the value from the original metadata file.
"""
data_dir = pathlib.Path(data_dir)
experiment_metadata_path = data_dir / 'experiment_metadata.json'
with experiment_metadata_path.open() as f:
experiment_metadata = json.load(f)
positions = experiment_metadata['positions']
new_positions = {}
names, xyzs = zip(*positions.items())
old_xys = []
new_xys = []
rotation = numpy.eye(2)
translation = [0, 0]
position_order = _farthest_points(numpy.array(xyzs)[:, :2])
viewer = scope_viewer_widget.ScopeViewerWidget(scope)
viewer.show()
viewer.layers.append(layer.Layer())
viewer.layers[0].tint = 0, 1, 0 # green
viewer.layers[1].tint = 1, 0, 1 # purple
with scope.camera.in_state(live_mode=True), scope.tl.lamp.in_state(enabled=True), scope.stage.in_state(async_=True):
for i in position_order:
position_name = names[i]
x, y, z = xyzs[i]
# now apply current guess of rotation and translation to get estimated
# new position
x, y = numpy.dot([x, y], rotation) + translation
scope.stage.position = x, y, z
position_dir = data_dir / position_name
position_metadata_path = position_dir / 'position_metadata.json'
with position_metadata_path.open() as f:
position_metadata = json.load(f)
if not ignore_autofocus_z:
for m in position_metadata[::-1]:
if 'fine_z' in m:
z = m['fine_z']
break
image = None
for m in position_metadata[::-1]:
timepoint = m['timepoint']
image_f = position_dir / f'{timepoint} {image_type}.png'
if image_f.exists():
image = freeimage.read(image_f)
break
scope.stage.wait()
if image is None:
print(f'Could not find any {image_type} image for position {position_name}; skipping')
continue
if len(viewer.live_image_page) == 2:
viewer.live_image_page[1] = image
else:
viewer.live_image_page.append(image)
out = viewer.input('refine position {} (press enter when done, or press "x" if the position is no longer available)'.format(position_name))
if out.strip().lower() == 'x':
continue
old_xys.append(x, y)
x, y, z = scope.stage.position
new_positions[position_name] = x, y, z
new_xys.append(x, y)
rotation, _, translation, _ = fit_affine.fit_affine(old_xys, new_xys, find_scale=False, find_translation=True)
angle = numpy.arctan2(rotation[0, 1], rotation[0, 0]) * 180 / numpy.pi
micron_trans = translation * 1000
print(f'\t estimated rotation angle: {angle:.1f}°')
print(f'\t estimated translation: ({micron_trans[0]:.0f}, {micron_trans[1]:.0f}) µm')
new_z = {position_name: z for position_name, (x, y, z) in new_positions.items()}
experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z
experiment_metadata['old_positions'] = positions
experiment_metadata['positions'] = new_positions
if not dry_run:
datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def write_metadata(self):
"""Write the current metadata dictionary back to the metadata file."""
datafile.json_encode_atomic_legible_to_file(self.metadata,
self.metadata_file)
def write_metadata(metadata, experiment_root):
"""Write experiment metadata file from a dictionary."""
metadata_file = pathlib.Path(experiment_root) / 'experiment_metadata.json'
datafile.json_encode_atomic_legible_to_file(metadata, metadata_file)
def _write_atomic_json(self, out_path, data):
datafile.json_encode_atomic_legible_to_file(data, out_path)
def remove_timepoint_for_position(experiment_root,
position,
timepoint,
dry_run=False):
"""Removes all memory of a given timepoint for a given position (files, metadata, annotations entries)
Parameters:
experiment_root: str/pathlib.Path pointing to experiment directory
position: str for position to modify
timepoint: str with standard format for timepoint acquisition (YYYY-MM-DDtHHMM)
dry_run: bool flag that toggles taking action (if False, only specifies when offending files are found)
Example Usage:
experiment_root = /path/to/experiment
position = '000'
timepoint = '2018-07-30t1200'
dry_run = True # No deleting; just see what happens
remove_timepoint_for_position(experiment_root, position, timepoint, dry_run=dry_run)
"""
experiment_root = pathlib.Path(experiment_root)
image_files_to_remove = [
img_file for img_file in (experiment_root / position).iterdir()
if img_file.name.startswith(timepoint)
]
if len(image_files_to_remove) > 0:
print(
f'Found image files for removal for position {position}: {image_files_to_remove}'
)
if not dry_run:
[img_file.unlink() for img_file in image_files_to_remove]
if (experiment_root / 'derived_data' / 'mask' / position).exists():
mask_files_to_remove = [
mask_file for mask_file in (experiment_root / 'derived_data' /
'mask' / position).iterdir()
if mask_file.name.startswith(timepoint)
]
if len(mask_files_to_remove) > 0:
print(
f'Found mask files for removal for position {position}: {mask_files_to_remove}'
)
if not dry_run:
[mask_file.unlink() for mask_file in mask_files_to_remove]
md_file = (experiment_root / position / 'position_metadata.json')
with md_file.open() as md_fp:
pos_md = json.load(md_fp)
acquired_timepoints = [
timepoint_info['timepoint'] for timepoint_info in pos_md
]
try:
offending_timepoint_index = acquired_timepoints.index(timepoint)
print(
f'Found entry for removal in position_metadata for position {position}'
)
if not dry_run:
del pos_md[offending_timepoint_index]
datafile.json_encode_atomic_legible_to_file(
pos_md, md_file) # Write out new position_metadata
except ValueError: # bad timepoint wasn't found
pass
position_annotation_file = experiment_root / 'annotations' / f'{position}.pickle'
if position_annotation_file.exists():
general_annotations, timepoint_annotations = load_data.read_annotation_file(
position_annotation_file)
if timepoint in timepoint_annotations:
print(
f'Found entry for removal in annotation for position {position}'
)
if not dry_run:
del timepoint_annotations[timepoint]
load_data.write_annotation_file(position_annotation_file,
general_annotations,
timepoint_annotations)