def run(self):
self.logger.name = type(self).__name__
self.logger.setLevel(self.args["log_level"])
ophys_etl_commit_sha = os.environ.get("OPHYS_ETL_COMMIT_SHA",
"local build")
self.logger.info(f"OPHYS_ETL_COMMIT_SHA: {ophys_etl_commit_sha}")
t0 = time.time()
video_path = pathlib.Path(self.args["video_path"])
exp_id = video_path.name.split("_")[0]
roi_path = pathlib.Path(self.args["roi_path"])
graph_path = pathlib.Path(self.args["graph_path"])
proj = get_max_and_avg(video_path)
self.logger.info("Calculated mean and max images...")
avg_img = proj["avg"]
max_img = proj["max"]
corr_img = graph_to_img(graph_path)
self.logger.info("Calculated correlation image...")
quantiles = (self.args["low_quantile"], self.args["high_quantile"])
q0, q1 = np.quantile(max_img, quantiles)
max_img = normalize_array(array=max_img,
lower_cutoff=q0,
upper_cutoff=q1)
q0, q1 = np.quantile(avg_img, quantiles)
avg_img = normalize_array(array=avg_img,
lower_cutoff=q0,
upper_cutoff=q1)
q0, q1 = np.quantile(corr_img, quantiles)
corr_img = normalize_array(array=corr_img,
lower_cutoff=q0,
upper_cutoff=q1)
self.logger.info("Normalized images...")
with open(roi_path, "rb") as in_file:
extract_roi_list = sanitize_extract_roi_list(json.load(in_file))
selected_rois = self.args['selected_rois']
if selected_rois is None:
selected_rois = [roi['id'] for roi in extract_roi_list]
selected_rois = set(selected_rois)
self.logger.info("Creating and writing ROI artifacts...")
for roi in extract_roi_list:
if roi['id'] not in selected_rois:
continue
self._write_thumbnails(extract_roi=roi,
max_img=max_img,
avg_img=avg_img,
corr_img=corr_img,
exp_id=exp_id)
self.logger.info(f"Created ROI artifacts in {time.time()-t0:.0f} "
"seconds.")
def add_list_of_roi_contours_to_img(
img: np.ndarray,
roi_list: Union[List[OphysROI], List[Dict]],
color: Union[Tuple[int, int, int],
Dict[int, Tuple[int, int, int]]] = (255, 0, 0),
alpha: float = 0.25) -> np.ndarray:
"""
Add colored ROI contours to an image
Parameters
----------
img: np.ndarray
RGB representation of image
roi_list: List[OphysROI]
list of ROIs to add to image
color: Union[Tuple[int,int, int],
Dict[int, Tuple[int, int, int]]
Either a representing an RGB color, or a dict
mapping roi_id to tuples representing RGB colors
(default = (255, 0, 0))
alpha: float
transparency factor to apply to ROI (default=0.25)
Returns
-------
new_img: np.ndarray
New image with ROI borders superimposed
"""
new_img = np.copy(img)
if len(roi_list) == 0:
return new_img
if not isinstance(roi_list[0], OphysROI):
roi_list = sanitize_extract_roi_list(roi_list)
roi_list = [extract_roi_to_ophys_roi(roi)
for roi in roi_list]
if isinstance(color, tuple):
color_map = {roi.roi_id: color for roi in roi_list}
else:
color_map = color
for roi in roi_list:
new_img = add_roi_contour_to_img(
new_img,
roi,
color_map[roi.roi_id],
alpha)
return new_img
def test_sanitize_extract_roi_list(suite2p_roi_fixture):
with open(suite2p_roi_fixture, 'rb') as in_file:
raw_roi_list = json.load(in_file)
assert len(raw_roi_list) > 0
extract_roi_list = sanitize_extract_roi_list(raw_roi_list)
assert len(extract_roi_list) == len(raw_roi_list)
for raw_roi, extract_roi in zip(raw_roi_list, extract_roi_list):
assert 'mask_matrix' not in extract_roi
assert 'valid_roi' not in extract_roi
assert 'roi_id' not in extract_roi
for e_key, r_key in (('id', 'roi_id'), ('mask', 'mask_matrix'),
('valid', 'valid_roi'), ('x', 'x'), ('y', 'y'),
('width', 'width'), ('height', 'height')):
assert extract_roi[e_key] == raw_roi[r_key]
def test_get_traces(classifier2021_video_fixture, suite2p_roi_fixture):
video_path = classifier2021_video_fixture
roi_path = suite2p_roi_fixture
with open(roi_path, 'rb') as in_file:
raw_roi_list = json.load(in_file)
extract_roi_list = sanitize_extract_roi_list(raw_roi_list)
ophys_roi_list = [
extract_roi_to_ophys_roi(roi) for roi in extract_roi_list
]
found_traces = get_traces(video_path, ophys_roi_list)
assert len(found_traces) == len(extract_roi_list)
with h5py.File(video_path, 'r') as in_file:
video_data = in_file['data'][()]
assert len(extract_roi_list) > 0
for roi in extract_roi_list:
assert roi['id'] in found_traces
expected_trace = np.zeros(video_data.shape[0], dtype=float)
npix = 0
r0 = roi['y']
c0 = roi['x']
for ir in range(roi['height']):
for ic in range(roi['width']):
if not roi['mask'][ir][ic]:
continue
npix += 1
row = r0 + ir
col = c0 + ic
expected_trace += video_data[:, row, col]
expected_trace = expected_trace / npix
np.testing.assert_array_equal(expected_trace, found_traces[roi['id']])
def test_sanitize_extract_roi_list(example_ophys_roi_list_fixture,
munge_mapping, extra_field):
extract_roi_list = [
rois_utils.ophys_roi_to_extract_roi(roi)
for roi in example_ophys_roi_list_fixture
]
if extra_field:
for ii, roi in enumerate(extract_roi_list):
roi['nonsense'] = ii
for roi in extract_roi_list:
assert 'nonsense' in roi
bad_roi_list = []
for roi in extract_roi_list:
new_roi = copy.deepcopy(roi)
for k in munge_mapping:
new_roi[munge_mapping[k]] = new_roi.pop(k)
bad_roi_list.append(new_roi)
assert bad_roi_list != extract_roi_list
cleaned_roi_list = rois_utils.sanitize_extract_roi_list(bad_roi_list)
assert cleaned_roi_list == extract_roi_list
def test_labeler_artifact_generator(
tmp_path_factory, classifier2021_video_fixture,
classifier2021_video_hash_fixture, suite2p_roi_fixture,
suite2p_roi_hash_fixture, classifier2021_corr_graph_fixture,
classifier2021_corr_graph_hash_fixture,
classifier2021_corr_png_fixture, classifier2021_corr_png_hash_fixture,
video_lower_quantile, video_upper_quantile, projection_lower_quantile,
projection_upper_quantile, use_graph, with_motion_border):
"""
Test that LabelerArtifactGenerator runs and produces expected output
"""
tmpdir = tmp_path_factory.mktemp('full_artifact_generation')
if with_motion_border:
motion_path = pathlib.Path(
tempfile.mkstemp(dir=tmpdir, suffix='.csv')[1])
with open(motion_path, 'w') as out_file:
out_file.write('x,y\n')
out_file.write('5,6\n')
out_file.write('14,-3\n')
expected_motion_border = {
'bottom': 6.0,
'top': 3.0,
'right_side': 14.0,
'left_side': 0.0
}
motion_path = str(motion_path.resolve().absolute())
else:
motion_path = None
expected_motion_border = {
'top': 0,
'bottom': 0,
'left_side': 0,
'right_side': 0
}
if use_graph:
corr_fixture = classifier2021_corr_graph_fixture
corr_hash = classifier2021_corr_graph_hash_fixture
else:
corr_fixture = classifier2021_corr_png_fixture
corr_hash = classifier2021_corr_png_hash_fixture
output_tuple = tempfile.mkstemp(dir=tmpdir,
prefix='artifact_file_',
suffix='.h5')
# without this, got a "too many files open" error
os.close(output_tuple[0])
output_path = pathlib.Path(output_tuple[1])
# because tempfile.mkstemp actually creates the file
output_path.unlink()
input_data = dict()
input_data['video_path'] = str(classifier2021_video_fixture)
input_data['roi_path'] = str(suite2p_roi_fixture)
input_data['correlation_path'] = str(corr_fixture)
input_data['artifact_path'] = str(output_path)
input_data['clobber'] = False
input_data['video_lower_quantile'] = video_lower_quantile
input_data['video_upper_quantile'] = video_upper_quantile
input_data['projection_lower_quantile'] = projection_lower_quantile
input_data['projection_upper_quantile'] = projection_upper_quantile
input_data['motion_border_path'] = motion_path
generator = LabelerArtifactGenerator(input_data=input_data, args=[])
generator.run()
assert output_path.is_file()
with h5py.File(output_path, 'r') as artifact_file:
motion_border = json.loads(
artifact_file['motion_border'][()].decode('utf-8'))
assert motion_border == expected_motion_border
# test that ROIs were written correctly
with open(suite2p_roi_fixture, 'rb') as in_file:
expected_rois = json.load(in_file)
expected_rois = sanitize_extract_roi_list(expected_rois)
artifact_rois = json.loads(artifact_file['rois'][()].decode('utf-8'))
assert expected_rois == artifact_rois
# test that all ROIs appear in color map
color_map = json.loads(
artifact_file['roi_color_map'][()].decode('utf-8'))
assert len(color_map) == len(expected_rois)
for roi in expected_rois:
assert str(roi['id']) in color_map
# test that traces were written correctly
ophys_rois = [extract_roi_to_ophys_roi(roi) for roi in expected_rois]
expected_traces = get_traces(classifier2021_video_fixture, ophys_rois)
for roi_id in expected_traces:
np.testing.assert_array_equal(
expected_traces[roi_id], artifact_file[f'traces/{roi_id}'][()])
# test that the scaled video data was written correctly
assert artifact_file['video_data'].chunks is not None
scaled_video = artifact_file['video_data'][()]
with h5py.File(classifier2021_video_fixture, 'r') as raw_file:
raw_video = raw_file['data'][()]
raw_max = np.max(raw_video, axis=0)
raw_avg = np.mean(raw_video, axis=0)
mn, mx = np.quantile(raw_video,
(video_lower_quantile, video_upper_quantile))
raw_video = np.where(raw_video > mn, raw_video, mn)
raw_video = np.where(raw_video < mx, raw_video, mx)
delta = mx - mn
raw_video = raw_video - mn
raw_video = raw_video.astype(float)
raw_video = np.round(255.0 * raw_video / delta).astype(np.uint8)
np.testing.assert_array_equal(raw_video, scaled_video)
del raw_video
del scaled_video
# test that max and avg projection images wer written correctly
for raw_img, img_key in zip((raw_max, raw_avg),
('max_projection', 'avg_projection')):
artifact_img = artifact_file[img_key][()]
mn, mx = np.quantile(
raw_img,
(projection_lower_quantile, projection_upper_quantile))
raw_img = np.where(raw_img > mn, raw_img, mn)
raw_img = np.where(raw_img < mx, raw_img, mx)
raw_img = raw_img.astype(float)
np.testing.assert_array_equal(raw_img, artifact_img)
artifact_corr = artifact_file['correlation_projection'][()]
if use_graph:
expected_corr = normalize_array(
graph_to_img(corr_fixture,
attribute_name='filtered_hnc_Gaussian'))
else:
expected_corr = normalize_array(
np.array(PIL.Image.open(corr_fixture, 'r')))
np.testing.assert_array_equal(artifact_corr, expected_corr)
metadata = json.loads(artifact_file['metadata'][()].decode('utf-8'))
# test that metadata has the right contents
assert metadata['video']['path'] == str(classifier2021_video_fixture)
assert metadata['video']['hash'] == classifier2021_video_hash_fixture
assert metadata['rois']['path'] == str(suite2p_roi_fixture)
assert metadata['rois']['hash'] == suite2p_roi_hash_fixture
assert metadata['correlation']['path'] == str(corr_fixture)
assert metadata['correlation']['hash'] == corr_hash
assert metadata['generator_args'] == input_data
if with_motion_border:
assert 'motion_csv' in metadata
else:
assert 'motion_csv' not in metadata
tmpdir = pathlib.Path(tmpdir)
path_list = [n for n in tmpdir.rglob('*')]
for this_path in path_list:
if this_path.is_file():
try:
this_path.unlink()
except Exception:
pass
def run(self):
video_path = pathlib.Path(self.args['video_path'])
correlation_path = pathlib.Path(self.args['correlation_path'])
roi_path = pathlib.Path(self.args['roi_path'])
if self.args['motion_border_path'] is not None:
motion_border_path = pathlib.Path(self.args['motion_border_path'])
max_shifts = get_max_correction_from_file(
input_csv=motion_border_path)
motion_border = motion_border_from_max_shift(max_shifts)
else:
motion_border_path = None
motion_border = MotionBorder(left_side=0,
right_side=0,
top=0,
bottom=0)
output_path = pathlib.Path(self.args['artifact_path'])
with open(roi_path, 'rb') as in_file:
raw_rois = json.load(in_file)
extract_roi_list = sanitize_extract_roi_list(raw_rois)
ophys_roi_list = [
extract_roi_to_ophys_roi(roi) for roi in extract_roi_list
]
logger.info("read ROIs")
trace_lookup = get_traces(video_path, ophys_roi_list)
logger.info("wrote traces")
metadata = create_metadata(input_args=self.args,
video_path=video_path,
roi_path=roi_path,
correlation_path=correlation_path,
motion_csv_path=motion_border_path)
logger.info("hashed all input files")
color_map = get_roi_color_map(ophys_roi_list)
logger.info("computed color map")
(avg_img_data, max_img_data) = create_max_and_avg_projections(
video_path, self.args['projection_lower_quantile'],
self.args['projection_upper_quantile'])
logger.info("Created max and avg projection images")
correlation_img_data = create_correlation_projection(correlation_path)
logger.info("Created correlation image")
scaled_video = read_and_scale(
video_path=pathlib.Path(video_path),
origin=(0, 0),
frame_shape=max_img_data.shape,
quantiles=(self.args['video_lower_quantile'],
self.args['video_upper_quantile']))
logger.info("Created scaled video")
# determine chunks in which to save the video data
ntime = scaled_video.shape[0]
nrows = scaled_video.shape[1]
ncols = scaled_video.shape[2]
video_chunks = (max(1, ntime // 10), max(1, nrows // 16),
max(1, ncols // 16))
with h5py.File(output_path, 'a') as out_file:
out_file.create_dataset(
'metadata',
data=json.dumps(metadata, sort_keys=True).encode('utf-8'))
out_file.create_dataset(
'rois', data=json.dumps(extract_roi_list).encode('utf-8'))
out_file.create_dataset('roi_color_map',
data=json.dumps(color_map).encode('utf-8'))
out_file.create_dataset('max_projection', data=max_img_data)
out_file.create_dataset('avg_projection', data=avg_img_data)
out_file.create_dataset('correlation_projection',
data=correlation_img_data)
out_file.create_dataset('video_data',
data=scaled_video,
chunks=video_chunks)
# note the transposition below;
# if you shift up, the suspect pixels are those that wrap
# on the bottom; if you shift right, the suspect pixels
# are those that wrap on the right, etc.
out_file.create_dataset(
'motion_border',
data=json.dumps(
{
'bottom': motion_border.bottom,
'top': motion_border.top,
'left_side': motion_border.left_side,
'right_side': motion_border.right_side
},
indent=2).encode('utf-8'))
trace_group = out_file.create_group('traces')
for roi_id in trace_lookup:
trace_group.create_dataset(str(roi_id),
data=trace_lookup[roi_id])
logger.info("Wrote all artifacts")