def update_z_positions(data_dir, scope): """Interactively update the z positions for an existing experiment. New positions are written to a 'z_updates' metadata dictionary, which is used by the acquisiton script to override the previous focal position. Parameters: data_dir: experiment directory with existing experiment_metadata.json file. scope: scope object. """ data_dir = pathlib.Path(data_dir) experiment_metadata_path = data_dir / 'experiment_metadata.json' with experiment_metadata_path.open() as f: experiment_metadata = json.load(f) positions = experiment_metadata['positions'] new_z = {} for position_name, (x,y,z) in sorted(positions.items()): position_dir = data_dir / position_name position_metadata_path = position_dir / 'position_metadata.json' with position_metadata_path.open() as f: position_metadata = json.load(f) for m in position_metadata[::-1]: if 'fine_z' in m: z = m['fine_z'] break scope.stage.position = x, y, z input('refine position {} (press enter when done)'.format(position_name)) new_z[position_name] = scope.stage.z experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def prune_z(experiment_dir): ''' For when I needed to rescue some scripts after Zach's update to new code ''' experiment_dir = pathlib.Path(experiment_dir) for position_dir in sorted(list(experiment_dir.iterdir())): if position_dir.is_dir() and position_dir.parts[-1][0].isnumeric(): shutil.copyfile(str(position_dir/'position_metadata.json'), str(position_dir/'position_metadata_old.json')) position_metadata = json.load((position_dir/'position_metadata.json').open('r')) if type(position_metadata[-1]['fine_z']) is list: position_metadata[-1]['fine_z'] = position_metadata[-1]['fine_z'][0] datafile.json_encode_atomic_legible_to_file(position_metadata, (position_dir/'position_metadata.json'))
def write_metadata(self): """Write the current set of metadata for all timepoints back to the metadata file.""" if self._metadata is None: self._load_metadata() # read from self._metadata instead of gathering from all timepoints in self # in case some timepoints have been filtered just temporarily -- don't want to # blow away their metadata metadata_list = [ dict(timepoint=name, **rest) for name, rest in sorted(self._metadata.items()) ] datafile.json_encode_atomic_legible_to_file(metadata_list, self.metadata_file)
def update_z_positions(data_dir, scope): """Interactively update the z positions for an existing experiment. New positions are written to a 'z_updates' metadata dictionary, which is used by the acquisiton script to override the previous focal position. Parameters: data_dir: experiment directory with existing experiment_metadata.json file. scope: scope object. """ data_dir = pathlib.Path(data_dir) experiment_metadata_path = data_dir / 'experiment_metadata.json' with experiment_metadata_path.open() as f: experiment_metadata = json.load(f) positions = experiment_metadata['positions'] new_z = {} for position_name, (x, y, z) in sorted(positions.items()): position_dir = data_dir / position_name position_metadata_path = position_dir / 'position_metadata.json' with position_metadata_path.open() as f: position_metadata = json.load(f) for m in position_metadata[::-1]: if 'fine_z' in m: z = m['fine_z'] break scope.stage.position = x, y, z input( 'refine position {} (press enter when done)'.format(position_name)) new_z[position_name] = scope.stage.z experiment_metadata.setdefault( 'z_updates', {})[datetime.datetime.now().isoformat()] = new_z datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def update_z_positions(data_dir, scope): """Interactively update the z positions for an existing experiment. New positions are written to a 'z_updates' metadata dictionary, which is used by the acquisiton script to override the previous focal position. Parameters: data_dir: experiment directory with existing experiment_metadata.json file. scope: scope object. """ data_dir = pathlib.Path(data_dir) experiment_metadata_path, experiment_metadata = metadata_util.get_experiment_metadata(data_dir) positions = experiment_metadata['positions'] new_z = {} for position_name in sorted(positions.keys()): position_dir, metadata_path, position_metadata = metadata_util.get_position_metadata(data_dir position_name) scope.stage.position = metadata_util.get_latest_coordinates(position_name, experiment_metadata, position_metadata) input('refine position {} (press enter when done)'.format(position_name)) new_z[position_name] = scope.stage.z experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def _write_atomic_json(self, out_path, data): datafile.json_encode_atomic_legible_to_file(data, out_path)
def update_positions(data_dir, scope, image_type='bf', dry_run=False, ignore_autofocus_z=False): """Interactively update the xyz positions for an existing experiment. New z positions are written to a 'z_updates' metadata dictionary, which is used by the acquisiton script to override the previous focal position. New x,y positions are written directly to the experiment metadata. (The old positions are stored in the metadata file as "old_positions".) Parameters: data_dir: experiment directory with existing experiment_metadata.json file. scope: scope object dry_run: if True, don't actually write files ignore_autofocus_z: if True, don't use the last-autofocused z value; instead just use the value from the original metadata file. """ data_dir = pathlib.Path(data_dir) experiment_metadata_path = data_dir / 'experiment_metadata.json' with experiment_metadata_path.open() as f: experiment_metadata = json.load(f) positions = experiment_metadata['positions'] new_positions = {} names, xyzs = zip(*positions.items()) old_xys = [] new_xys = [] rotation = numpy.eye(2) translation = [0, 0] position_order = _farthest_points(numpy.array(xyzs)[:, :2]) viewer = scope_viewer_widget.ScopeViewerWidget(scope) viewer.show() viewer.layers.append(layer.Layer()) viewer.layers[0].tint = 0, 1, 0 # green viewer.layers[1].tint = 1, 0, 1 # purple with scope.camera.in_state(live_mode=True), scope.tl.lamp.in_state( enabled=True), scope.stage.in_state(async_=True): for i in position_order: position_name = names[i] x, y, z = xyzs[i] # now apply current guess of rotation and translation to get estimated # new position x, y = numpy.dot([x, y], rotation) + translation scope.stage.position = x, y, z position_dir = data_dir / position_name position_metadata_path = position_dir / 'position_metadata.json' with position_metadata_path.open() as f: position_metadata = json.load(f) if not ignore_autofocus_z: for m in position_metadata[::-1]: if 'fine_z' in m: z = m['fine_z'] break image = None for m in position_metadata[::-1]: timepoint = m['timepoint'] image_f = position_dir / f'{timepoint} {image_type}.png' if image_f.exists(): image = freeimage.read(image_f) break scope.stage.wait() if image is None: print( f'Could not find any {image_type} image for position {position_name}; skipping' ) continue if len(viewer.live_image_page) == 2: viewer.live_image_page[1] = image else: viewer.live_image_page.append(image) out = viewer.input( 'refine position {} (press enter when done, or press "x" if the position is no longer available)' .format(position_name)) if out.strip().lower() == 'x': continue old_xys.append(x, y) x, y, z = scope.stage.position new_positions[position_name] = x, y, z new_xys.append(x, y) rotation, _, translation, _ = fit_affine.fit_affine( old_xys, new_xys, find_scale=False, find_translation=True) angle = numpy.arctan2(rotation[0, 1], rotation[0, 0]) * 180 / numpy.pi micron_trans = translation * 1000 print(f'\t estimated rotation angle: {angle:.1f}°') print( f'\t estimated translation: ({micron_trans[0]:.0f}, {micron_trans[1]:.0f}) µm' ) new_z = { position_name: z for position_name, (x, y, z) in new_positions.items() } experiment_metadata.setdefault( 'z_updates', {})[datetime.datetime.now().isoformat()] = new_z experiment_metadata['old_positions'] = positions experiment_metadata['positions'] = new_positions if not dry_run: datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def update_positions(data_dir, scope, image_type='bf', dry_run=False, ignore_autofocus_z=False): """Interactively update the xyz positions for an existing experiment. New z positions are written to a 'z_updates' metadata dictionary, which is used by the acquisiton script to override the previous focal position. New x,y positions are written directly to the experiment metadata. (The old positions are stored in the metadata file as "old_positions".) Parameters: data_dir: experiment directory with existing experiment_metadata.json file. scope: scope object dry_run: if True, don't actually write files ignore_autofocus_z: if True, don't use the last-autofocused z value; instead just use the value from the original metadata file. """ data_dir = pathlib.Path(data_dir) experiment_metadata_path = data_dir / 'experiment_metadata.json' with experiment_metadata_path.open() as f: experiment_metadata = json.load(f) positions = experiment_metadata['positions'] new_positions = {} names, xyzs = zip(*positions.items()) old_xys = [] new_xys = [] rotation = numpy.eye(2) translation = [0, 0] position_order = _farthest_points(numpy.array(xyzs)[:, :2]) viewer = scope_viewer_widget.ScopeViewerWidget(scope) viewer.show() viewer.layers.append(layer.Layer()) viewer.layers[0].tint = 0, 1, 0 # green viewer.layers[1].tint = 1, 0, 1 # purple with scope.camera.in_state(live_mode=True), scope.tl.lamp.in_state(enabled=True), scope.stage.in_state(async_=True): for i in position_order: position_name = names[i] x, y, z = xyzs[i] # now apply current guess of rotation and translation to get estimated # new position x, y = numpy.dot([x, y], rotation) + translation scope.stage.position = x, y, z position_dir = data_dir / position_name position_metadata_path = position_dir / 'position_metadata.json' with position_metadata_path.open() as f: position_metadata = json.load(f) if not ignore_autofocus_z: for m in position_metadata[::-1]: if 'fine_z' in m: z = m['fine_z'] break image = None for m in position_metadata[::-1]: timepoint = m['timepoint'] image_f = position_dir / f'{timepoint} {image_type}.png' if image_f.exists(): image = freeimage.read(image_f) break scope.stage.wait() if image is None: print(f'Could not find any {image_type} image for position {position_name}; skipping') continue if len(viewer.live_image_page) == 2: viewer.live_image_page[1] = image else: viewer.live_image_page.append(image) out = viewer.input('refine position {} (press enter when done, or press "x" if the position is no longer available)'.format(position_name)) if out.strip().lower() == 'x': continue old_xys.append(x, y) x, y, z = scope.stage.position new_positions[position_name] = x, y, z new_xys.append(x, y) rotation, _, translation, _ = fit_affine.fit_affine(old_xys, new_xys, find_scale=False, find_translation=True) angle = numpy.arctan2(rotation[0, 1], rotation[0, 0]) * 180 / numpy.pi micron_trans = translation * 1000 print(f'\t estimated rotation angle: {angle:.1f}°') print(f'\t estimated translation: ({micron_trans[0]:.0f}, {micron_trans[1]:.0f}) µm') new_z = {position_name: z for position_name, (x, y, z) in new_positions.items()} experiment_metadata.setdefault('z_updates', {})[datetime.datetime.now().isoformat()] = new_z experiment_metadata['old_positions'] = positions experiment_metadata['positions'] = new_positions if not dry_run: datafile.json_encode_atomic_legible_to_file(experiment_metadata, experiment_metadata_path)
def write_metadata(self): """Write the current metadata dictionary back to the metadata file.""" datafile.json_encode_atomic_legible_to_file(self.metadata, self.metadata_file)
def write_metadata(metadata, experiment_root): """Write experiment metadata file from a dictionary.""" metadata_file = pathlib.Path(experiment_root) / 'experiment_metadata.json' datafile.json_encode_atomic_legible_to_file(metadata, metadata_file)
def _write_atomic_json(self, out_path, data): datafile.json_encode_atomic_legible_to_file(data, out_path)
def remove_timepoint_for_position(experiment_root, position, timepoint, dry_run=False): """Removes all memory of a given timepoint for a given position (files, metadata, annotations entries) Parameters: experiment_root: str/pathlib.Path pointing to experiment directory position: str for position to modify timepoint: str with standard format for timepoint acquisition (YYYY-MM-DDtHHMM) dry_run: bool flag that toggles taking action (if False, only specifies when offending files are found) Example Usage: experiment_root = /path/to/experiment position = '000' timepoint = '2018-07-30t1200' dry_run = True # No deleting; just see what happens remove_timepoint_for_position(experiment_root, position, timepoint, dry_run=dry_run) """ experiment_root = pathlib.Path(experiment_root) image_files_to_remove = [ img_file for img_file in (experiment_root / position).iterdir() if img_file.name.startswith(timepoint) ] if len(image_files_to_remove) > 0: print( f'Found image files for removal for position {position}: {image_files_to_remove}' ) if not dry_run: [img_file.unlink() for img_file in image_files_to_remove] if (experiment_root / 'derived_data' / 'mask' / position).exists(): mask_files_to_remove = [ mask_file for mask_file in (experiment_root / 'derived_data' / 'mask' / position).iterdir() if mask_file.name.startswith(timepoint) ] if len(mask_files_to_remove) > 0: print( f'Found mask files for removal for position {position}: {mask_files_to_remove}' ) if not dry_run: [mask_file.unlink() for mask_file in mask_files_to_remove] md_file = (experiment_root / position / 'position_metadata.json') with md_file.open() as md_fp: pos_md = json.load(md_fp) acquired_timepoints = [ timepoint_info['timepoint'] for timepoint_info in pos_md ] try: offending_timepoint_index = acquired_timepoints.index(timepoint) print( f'Found entry for removal in position_metadata for position {position}' ) if not dry_run: del pos_md[offending_timepoint_index] datafile.json_encode_atomic_legible_to_file( pos_md, md_file) # Write out new position_metadata except ValueError: # bad timepoint wasn't found pass position_annotation_file = experiment_root / 'annotations' / f'{position}.pickle' if position_annotation_file.exists(): general_annotations, timepoint_annotations = load_data.read_annotation_file( position_annotation_file) if timepoint in timepoint_annotations: print( f'Found entry for removal in annotation for position {position}' ) if not dry_run: del timepoint_annotations[timepoint] load_data.write_annotation_file(position_annotation_file, general_annotations, timepoint_annotations)