def get_aligned_channels(ins, chn_coords, one, ba=None, save_dir=None):
ba = ba or AllenAtlas(25)
depths = chn_coords[:, 1]
xyz = np.array(ins['json']['xyz_picks']) / 1e6
traj = one.alyx.rest('trajectories', 'list', probe_insertion=ins['id'],
provenance='Ephys aligned histology track')[0]
align_key = ins['json']['extended_qc']['alignment_stored']
feature = traj['json'][align_key][0]
track = traj['json'][align_key][1]
ephysalign = EphysAlignment(xyz, depths, track_prev=track,
feature_prev=feature,
brain_atlas=ba, speedy=True)
channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6)
channels_atlas_ids = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
out_files = []
if save_dir is not None:
f_name = Path(save_dir).joinpath('channels.mlapdv.npy')
np.save(f_name, channels_mlapdv)
out_files.append(f_name)
f_name = Path(save_dir).joinpath('channels.brainLocationIds_ccf_2017.npy')
np.save(f_name, channels_atlas_ids)
out_files.append(f_name)
return out_files
def get_channels(self, traj, ins=None, depths=None):
if depths is None:
depths = SITES_COORDINATES[:, 1]
if traj['provenance'] == 'Planned' or traj['provenance'] == 'Micro-manipulator':
ins = atlas.Insertion.from_dict(traj)
# Deepest coordinate first
xyz = np.c_[ins.tip, ins.entry].T
xyz_channels = histology.interpolate_along_track(xyz, (depths +
TIP_SIZE_UM) / 1e6)
else:
if ins is None:
ins_idx = np.where(traj['probe_insertion'] == self.ins['ids'])[0][0]
xyz = np.array(self.ins['insertions'][ins_idx]['json']['xyz_picks']) / 1e6
else:
xyz = np.array(ins['json']['xyz_picks']) / 1e6
if traj['provenance'] == 'Histology track':
xyz = xyz[np.argsort(xyz[:, 2]), :]
xyz_channels = histology.interpolate_along_track(xyz, (depths +
TIP_SIZE_UM) / 1e6)
else:
if ins is None:
align_key = (self.ins['insertions'][ins_idx]['json']['extended_qc']
['alignment_stored'])
else:
align_key = ins['json']['extended_qc']['alignment_stored']
feature = traj['json'][align_key][0]
track = traj['json'][align_key][1]
ephysalign = EphysAlignment(xyz, depths, track_prev=track,
feature_prev=feature,
brain_atlas=self.ba, speedy=True)
xyz_channels = ephysalign.get_channel_locations(feature, track)
return xyz_channels
def compute_similarity_matrix(self):
"""
Computes the similarity matrix between each alignment stored in the ephys aligned
trajectory. Similarity matrix based on number of clusters that share brain region and
parent brain region
"""
r = regions_from_allen_csv()
clusters = dict()
for iK, key in enumerate(self.align_keys_sorted):
# Location of reference lines used for alignment
feature = np.array(self.alignments[key][0])
track = np.array(self.alignments[key][1])
# Instantiate EphysAlignment object
ephysalign = EphysAlignment(self.xyz_picks,
self.depths,
track_prev=track,
feature_prev=feature,
brain_atlas=self.brain_atlas)
# Find xyz location of all channels
xyz_channels = ephysalign.get_channel_locations(feature, track)
brain_regions = ephysalign.get_brain_locations(xyz_channels)
# Find the location of clusters along the alignment
cluster_info = dict()
cluster_info['brain_id'] = brain_regions['id'][self.cluster_chns]
cluster_info['parent_id'] = r.get(
ids=cluster_info['brain_id']).parent.astype(int)
clusters.update({key: cluster_info})
sim_matrix = np.zeros(
(len(self.align_keys_sorted), len(self.align_keys_sorted)))
for ik, key in enumerate(self.align_keys_sorted):
for ikk, key2 in enumerate(self.align_keys_sorted):
same_id = np.where(
clusters[key]['brain_id'] == clusters[key2]['brain_id'])[0]
not_same_id = \
np.where(clusters[key]['brain_id'] != clusters[key2]['brain_id'])[0]
same_parent = np.where(
clusters[key]['parent_id'][not_same_id] == clusters[key2]
['parent_id'][not_same_id])[0]
sim_matrix[ik, ikk] = len(same_id) + (len(same_parent) * 0.5)
# Normalise
sim_matrix_norm = sim_matrix / np.max(sim_matrix)
return sim_matrix_norm
class TestsEphysReconstruction(unittest.TestCase):
def setUp(self):
self.ephysalign = EphysAlignment(xyz_picks, track_prev=track_prev,
feature_prev=feature_prev, brain_atlas=brain_atlas)
self.feature = self.ephysalign.feature_init
self.track = self.ephysalign.track_init
def test_channel_locations(self):
xyz_channels = self.ephysalign.get_channel_locations(self.feature, self.track,
depths=depths)
self.assertTrue(np.all(np.isclose(xyz_channels[0, :], xyz_channels_ref[0])))
self.assertTrue(np.all(np.isclose(xyz_channels[-1, :], xyz_channels_ref[-1])))
brain_regions = self.ephysalign.get_brain_locations(xyz_channels)
self.assertTrue(np.all(np.equal(np.unique(brain_regions.acronym), brain_regions_ref)))
def get_channels(self, alf_object, collection):
electrodes = {}
try:
electrodes = self.one.load_object(self.eid, alf_object, collection=collection)
electrodes['axial_um'] = electrodes['localCoordinates'][:, 1]
except ALFObjectNotFound:
_logger.warning(f'{alf_object} does not yet exist')
if self.hist_lookup[self.histology_status] == 3:
try:
electrodes['atlas_id'] = electrodes['brainLocationIds_ccf_2017']
electrodes['mlapdv'] = electrodes['mlapdv'] / 1e6
except KeyError:
_logger.warning('Insertion resolved but brainLocationIds_ccf_2017 attribute do not exist')
if self.hist_lookup[self.histology_status] > 0 and 'atlas_id' not in electrodes.keys():
if not self.brain_atlas:
self.brain_atlas = AllenAtlas()
self.brain_regions = self.brain_regions or self.brain_atlas.regions
if 'localCoordinates' not in electrodes.keys():
geometry = trace_header(version=1)
electrodes['localCoordinates'] = np.c_[geometry['x'], geometry['y']]
electrodes['axial_um'] = electrodes['localCoordinates'][:, 1]
depths = electrodes['localCoordinates'][:, 1]
xyz = np.array(self.ins['json']['xyz_picks']) / 1e6
if self.hist_lookup[self.histology_status] >= 2:
traj = self.one.alyx.rest('trajectories', 'list', provenance='Ephys aligned histology track',
probe_insertion=self.pid)[0]
align_key = self.ins['json']['extended_qc']['alignment_stored']
feature = traj['json'][align_key][0]
track = traj['json'][align_key][1]
ephysalign = EphysAlignment(xyz, depths, track_prev=track,
feature_prev=feature,
brain_atlas=self.brain_atlas, speedy=True)
electrodes['mlapdv'] = ephysalign.get_channel_locations(feature, track)
electrodes['atlas_id'] = self.brain_atlas.regions.get(self.brain_atlas.get_labels(electrodes['mlapdv']))['id']
if self.hist_lookup[self.histology_status] == 1:
xyz = xyz[np.argsort(xyz[:, 2]), :]
electrodes['mlapdv'] = interpolate_along_track(xyz, (depths + TIP_SIZE_UM) / 1e6)
electrodes['atlas_id'] = self.brain_atlas.regions.get(self.brain_atlas.get_labels(electrodes['mlapdv']))['id']
return electrodes
def plot_scaling(region, scale, mapper, ax):
for reg, col in zip(region_scaled, scale_factor):
height = np.abs(reg[1] - reg[0])
color = np.array(mapper.to_rgba(col, bytes=True)) / 255
ax.bar(x=1.1, height=height, width=0.2, color=color, bottom=reg[0], edgecolor='w')
sec_ax = ax.secondary_yaxis('right')
sec_ax.set_yticks(np.mean(region, axis=1))
sec_ax.set_yticklabels(np.around(scale, 2))
sec_ax.tick_params(axis="y", direction="in")
sec_ax.set_ylim([20, 3840])
fig, ax = plt.subplots(1, len(alignments) + 1, figsize=(15, 15))
ephysalign = EphysAlignment(xyz_picks, depths, brain_atlas=brain_atlas)
feature, track, _ = ephysalign.get_track_and_feature()
channels_orig = ephysalign.get_channel_locations(feature, track)
region, region_label = ephysalign.scale_histology_regions(feature, track)
region_scaled, scale_factor = ephysalign.get_scale_factor(region)
region_colour = ephysalign.region_colour
norm = matplotlib.colors.Normalize(vmin=0.5, vmax=1.5, clip=True)
mapper = matplotlib.cm.ScalarMappable(norm=norm, cmap=matplotlib.cm.seismic)
ax_i = fig.axes[0]
plot_regions(region, region_label, region_colour, ax_i)
plot_scaling(region_scaled, scale_factor, mapper, ax_i)
ax_i.set_title('Original')
for iK, key in enumerate(alignments):
# Location of reference lines used for alignmnet
feature = np.array(alignments[key][0])
class TestsEphysAlignment(unittest.TestCase):
def setUp(self):
self.ephysalign = EphysAlignment(xyz_picks)
self.feature = self.ephysalign.feature_init
self.track = self.ephysalign.track_init
def test_no_scaling(self):
xyz_channels = self.ephysalign.get_channel_locations(self.feature,
self.track,
depths=depths)
coords = np.r_[[xyz_picks[-1, :]], [xyz_channels[0, :]]]
dist_to_fist_electrode = np.around(_cumulative_distance(coords)[-1], 5)
assert np.isclose(dist_to_fist_electrode, (TIP_SIZE_UM + 20) / 1e6)
def test_offset(self):
feature_val = 500 / 1e6
track_val = 1000 / 1e6
tracks = np.sort(np.r_[self.track[[0, -1]], track_val])
track_new = self.ephysalign.feature2track(tracks, self.feature,
self.track)
feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val])
track_new = self.ephysalign.adjust_extremes_uniform(
feature_new, track_new)
xyz_channels = self.ephysalign.get_channel_locations(feature_new,
track_new,
depths=depths)
coords = np.r_[[xyz_picks[-1, :]], [xyz_channels[0, :]]]
dist_to_fist_electrode = np.around(_cumulative_distance(coords)[-1], 5)
assert np.isclose(dist_to_fist_electrode,
((TIP_SIZE_UM + 20) / 1e6 + feature_val))
track_val = self.ephysalign.track2feature(track_val, feature_new,
track_new)
self.assertTrue(np.all(np.isclose(track_val, feature_val)))
region_new, _ = self.ephysalign.scale_histology_regions(
feature_new, track_new)
_, scale_factor = self.ephysalign.get_scale_factor(region_new)
self.assertTrue(np.all(np.isclose(scale_factor, 1)))
def test_uniform_scaling(self):
feature_val = np.array([500, 700, 2000]) / 1e6
track_val = np.array([1000, 1300, 2700]) / 1e6
tracks = np.sort(np.r_[self.track[[0, -1]], track_val])
track_new = self.ephysalign.feature2track(tracks, self.feature,
self.track)
feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val])
track_new = self.ephysalign.adjust_extremes_uniform(
feature_new, track_new)
region_new, _ = self.ephysalign.scale_histology_regions(
feature_new, track_new)
_, scale_factor = self.ephysalign.get_scale_factor(region_new)
self.assertTrue(np.isclose(scale_factor[0], 1))
self.assertTrue(np.isclose(scale_factor[-1], 1))
def test_linear_scaling(self):
feature_val = np.array([500, 700, 2000]) / 1e6
track_val = np.array([1000, 1300, 2700]) / 1e6
tracks = np.sort(np.r_[self.track[[0, -1]], track_val])
track_new = self.ephysalign.feature2track(tracks, self.feature,
self.track)
feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val])
fit = np.polyfit(feature_new[1:-1], track_new[1:-1], 1)
linear_fit = np.around(1 / fit[0], 3)
feature_new, track_new = self.ephysalign.adjust_extremes_linear(
feature_new, track_new, extend_feature=1)
region_new, _ = self.ephysalign.scale_histology_regions(
feature_new, track_new)
_, scale_factor = self.ephysalign.get_scale_factor(region_new)
self.assertTrue(np.isclose(np.around(scale_factor[0], 3), linear_fit))
self.assertTrue(np.isclose(np.around(scale_factor[-1], 3), linear_fit))
def upload_channels(self, alignment_key, upload_alyx, upload_flatiron):
"""
Upload channels to alyx and flatiron based on the alignment specified by the alignment key
"""
feature = np.array(self.alignments[alignment_key][0])
track = np.array(self.alignments[alignment_key][1])
ephysalign = EphysAlignment(self.xyz_picks,
self.depths,
track_prev=track,
feature_prev=feature,
brain_atlas=self.brain_atlas)
# Find the channels
channels_mlapdv = np.int32(
ephysalign.get_channel_locations(feature, track) * 1e6)
channels_brainID = ephysalign.get_brain_locations(channels_mlapdv /
1e6)['id']
# Find the clusters
r = regions_from_allen_csv()
clusters_mlapdv = channels_mlapdv[self.cluster_chns]
clusters_brainID = channels_brainID[self.cluster_chns]
clusters_brainAcro = r.get(ids=clusters_brainID).acronym
# upload datasets to flatiron
files_to_register = []
if upload_flatiron:
ftp_patcher = FTPPatcher(one=self.one)
insertion = self.one.alyx.rest('insertions', 'read', id=self.eid)
alf_path = self.one.path_from_eid(insertion['session']).joinpath(
'alf', insertion['name'])
alf_path.mkdir(exist_ok=True, parents=True)
# Make the channels.mlapdv dataset
f_name = alf_path.joinpath('channels.mlapdv.npy')
np.save(f_name, channels_mlapdv)
files_to_register.append(f_name)
# Make the channels.brainLocationIds dataset
f_name = alf_path.joinpath(
'channels.brainLocationIds_ccf_2017.npy')
np.save(f_name, channels_brainID)
files_to_register.append(f_name)
# Make the clusters.mlapdv dataset
f_name = alf_path.joinpath('clusters.mlapdv.npy')
np.save(f_name, clusters_mlapdv)
files_to_register.append(f_name)
# Make the clusters.brainLocationIds dataset
f_name = alf_path.joinpath(
'clusters.brainLocationIds_ccf_2017.npy')
np.save(f_name, clusters_brainID)
files_to_register.append(f_name)
# Make the clusters.brainLocationAcronym dataset
f_name = alf_path.joinpath(
'clusters.brainLocationAcronyms_ccf_2017.npy')
np.save(f_name, clusters_brainAcro)
files_to_register.append(f_name)
self.log.info("Writing datasets to FlatIron")
ftp_patcher.create_dataset(path=files_to_register,
created_by=self.one._par.ALYX_LOGIN)
# Need to change channels stored on alyx as well as the stored key is not the same as the
# latest key
if upload_alyx:
if alignment_key != self.align_keys_sorted[0]:
histology.register_aligned_track(self.eid,
channels_mlapdv / 1e6,
chn_coords=SITES_COORDINATES,
one=self.one,
overwrite=True,
channels=self.channels)
ephys_traj = self.one.alyx.rest(
'trajectories',
'list',
probe_insertion=self.eid,
provenance='Ephys aligned histology track')
patch_dict = {'json': self.alignments}
self.one.alyx.rest('trajectories',
'partial_update',
id=ephys_traj[0]['id'],
data=patch_dict)
return files_to_register
def upload_channels(self, alignment_key, upload_alyx, upload_flatiron):
"""
Upload channels to alyx and flatiron based on the alignment specified by the alignment key
"""
feature = np.array(self.alignments[alignment_key][0])
track = np.array(self.alignments[alignment_key][1])
try:
meta_dset = self.one.list_datasets(self.insertion['session'], '*ap.meta',
collection=f'raw_ephys_data/{self.insertion["name"]}')
meta_file = self.one.load_dataset(self.insertion['session'], meta_dset[0].split('/')[-1],
collection=f'raw_ephys_data/{self.insertion["name"]}',
download_only=True)
geometry = spikeglx.read_geometry(meta_file)
chns = np.c_[geometry['x'], geometry['y']]
except Exception as err:
self.log.warning(f"Could not compute channel locations from meta file, errored with message: {err}. "
f"Will use default Neuropixel 1 channels")
geometry = trace_header(version=1)
chns = np.c_[geometry['x'], geometry['y']]
ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1],
track_prev=track,
feature_prev=feature,
brain_atlas=self.brain_atlas)
channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6)
channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
# Need to change channels stored on alyx as well as the stored key is not the same as the latest key
if upload_alyx:
if alignment_key != self.align_keys_sorted[0]:
histology.register_aligned_track(self.eid, channels_mlapdv / 1e6,
chn_coords=chns, one=self.one,
overwrite=True, channels=self.channels_flag,
brain_atlas=self.brain_atlas)
ephys_traj = self.one.alyx.get(f'/trajectories?&probe_insertion={self.eid}'
'&provenance=Ephys aligned histology track',
clobber=True)
patch_dict = {'json': self.alignments}
self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'],
data=patch_dict)
files_to_register = []
if upload_flatiron:
ftp_patcher = FTPPatcher(one=self.one)
alf_path = self.one.eid2path(self.insertion['session']).joinpath('alf', self.insertion["name"])
alf_path.mkdir(exist_ok=True, parents=True)
f_name = alf_path.joinpath('electrodeSites.mlapdv.npy')
np.save(f_name, channels_mlapdv)
files_to_register.append(f_name)
f_name = alf_path.joinpath('electrodeSites.brainLocationIds_ccf_2017.npy')
np.save(f_name, channels_atlas_id)
files_to_register.append(f_name)
f_name = alf_path.joinpath('electrodeSites.localCoordinates.npy')
np.save(f_name, chns)
files_to_register.append(f_name)
probe_collections = self.one.list_collections(self.insertion['session'], filename='channels*',
collection=f'alf/{self.insertion["name"]}*')
for collection in probe_collections:
chns = self.one.load_dataset(self.insertion['session'], 'channels.localCoordinates', collection=collection)
ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1],
track_prev=track,
feature_prev=feature,
brain_atlas=self.brain_atlas)
channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6)
channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id']
alf_path = self.one.eid2path(self.insertion['session']).joinpath(collection)
alf_path.mkdir(exist_ok=True, parents=True)
f_name = alf_path.joinpath('channels.mlapdv.npy')
np.save(f_name, channels_mlapdv)
files_to_register.append(f_name)
f_name = alf_path.joinpath('channels.brainLocationIds_ccf_2017.npy')
np.save(f_name, channels_atlas_id)
files_to_register.append(f_name)
self.log.info("Writing datasets to FlatIron")
ftp_patcher.create_dataset(path=files_to_register,
created_by=self.one.alyx.user)
return files_to_register
def plot_alignment(insertion, traj, ind=None):
depths = SITES_COORDINATES[:, 1]
xyz_picks = np.array(insertion['json']['xyz_picks']) / 1e6
alignments = traj['json'].copy()
k = list(alignments.keys())[-1] # if only I had a Walrus available !
alignments = {k: alignments[k]}
# Create a figure and arrange using gridspec
widths = [1, 2.5]
heights = [1] * len(alignments)
gs_kw = dict(width_ratios=widths, height_ratios=heights)
fig, axis = plt.subplots(len(alignments),
2,
constrained_layout=True,
gridspec_kw=gs_kw,
figsize=(8, 9))
# Iterate over all alignments for trajectory
# 1. Plot brain regions that channel pass through
# 2. Plot coronal slice along trajectory with location of channels shown as red points
# 3. Save results for each alignment into a dict - channels
channels = {}
for iK, key in enumerate(alignments):
# Location of reference lines used for alignmnet
feature = np.array(alignments[key][0])
track = np.array(alignments[key][1])
chn_coords = SITES_COORDINATES
# Instantiate EphysAlignment object
ephysalign = EphysAlignment(xyz_picks,
depths,
track_prev=track,
feature_prev=feature)
# Find xyz location of all channels
xyz_channels = ephysalign.get_channel_locations(feature, track)
# Find brain region that each channel is located in
brain_regions = ephysalign.get_brain_locations(xyz_channels)
# Add extra keys to store all useful information as one bunch object
brain_regions['xyz'] = xyz_channels
brain_regions['lateral'] = chn_coords[:, 0]
brain_regions['axial'] = chn_coords[:, 1]
# Store brain regions result in channels dict with same key as in alignment
channel_info = {key: brain_regions}
channels.update(channel_info)
# For plotting -> extract the boundaries of the brain regions, as well as CCF label and colour
region, region_label, region_colour, _ = ephysalign.get_histology_regions(
xyz_channels, depths)
# Make plot that shows the brain regions that channels pass through
ax_regions = fig.axes[iK * 2]
for reg, col in zip(region, region_colour):
height = np.abs(reg[1] - reg[0])
bottom = reg[0]
color = col / 255
ax_regions.bar(x=0.5,
height=height,
width=1,
color=color,
bottom=reg[0],
edgecolor='w')
ax_regions.set_yticks(region_label[:, 0].astype(int))
ax_regions.yaxis.set_tick_params(labelsize=8)
ax_regions.get_xaxis().set_visible(False)
ax_regions.set_yticklabels(region_label[:, 1])
ax_regions.spines['right'].set_visible(False)
ax_regions.spines['top'].set_visible(False)
ax_regions.spines['bottom'].set_visible(False)
ax_regions.hlines([0, 3840],
*ax_regions.get_xlim(),
linestyles='dashed',
linewidth=3,
colors='k')
# ax_regions.plot(np.ones(channel_depths_track.shape), channel_depths_track, '*r')
# Make plot that shows coronal slice that trajectory passes through with location of channels
# shown in red
ax_slice = fig.axes[iK * 2 + 1]
brain_atlas.plot_tilted_slice(xyz_channels, axis=1, ax=ax_slice)
ax_slice.plot(xyz_channels[:, 0] * 1e6, xyz_channels[:, 2] * 1e6, 'r*')
ax_slice.title.set_text(insertion['id'] + '\n' + str(key))
# Make sure the plot displays
plt.show()
class TestsAlignmentQcGUI(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
probe = ['probe00', 'probe01']
create_alyx_probe_insertions(session_path=EPHYS_SESSION,
model='3B2',
labels=probe,
one=one,
force=True)
cls.probe_id = one.alyx.rest('insertions',
'list',
session=EPHYS_SESSION,
name='probe00')[0]['id']
cls.probe_id2 = one.alyx.rest('insertions',
'list',
session=EPHYS_SESSION,
name='probe01')[0]['id']
data = np.load(Path(
Path(__file__).parent.joinpath('fixtures',
'data_alignmentqc_gui.npz')),
allow_pickle=True)
cls.xyz_picks = data['xyz_picks']
cls.alignments = data['alignments'].tolist()
cls.cluster_chns = data['cluster_chns']
register_track(cls.probe_id,
picks=cls.xyz_picks,
one=one,
overwrite=True,
channels=False)
register_track(cls.probe_id2,
picks=cls.xyz_picks,
one=one,
overwrite=True,
channels=False)
def setUp(self) -> None:
self.resolved_key = '2020-09-14T15:44:56_nate'
self.ld = LoadData(one=one,
brain_atlas=brain_atlas,
testing=True,
probe_id=self.probe_id)
_ = self.ld.get_xyzpicks()
self.ld.cluster_chns = self.cluster_chns
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
self.ephysalign = EphysAlignment(self.ld.xyz_picks,
self.ld.chn_depths,
brain_atlas=self.ld.brain_atlas)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
if traj:
self.prev_traj_id = traj[0]['id']
def test_alignments(self):
checks = getmembers(
TestsAlignmentQcGUI,
lambda x: isfunction(x) and re.match(r'^_\d{2}_.*', x.__name__))
# Run each function in order
for name, fn in sorted(checks, key=lambda x: x[0]):
if not name.startswith('_01_'):
self.setUp()
fn(self)
def _01_no_alignment(self):
prev_align = self.ld.get_previous_alignments()
assert (len(prev_align) == 1)
assert (prev_align[0] == 'original')
feature, track = self.ld.get_starting_alignment(0)
assert (not feature)
assert (not track)
assert (not self.ld.alignments)
assert (self.ld.resolved == 0)
def _02_one_alignment(self):
key = '2020-07-26T17:06:58_alejandro'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 1)
# Not added user evaluation so should just contain feature and track lists
assert (len(traj[0]['json'][key]) == 2)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['extended_qc']['alignment_count'] == 1)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (not insertion['json']['extended_qc'].get('experimenter', None))
assert (insertion['json']['qc'] == 'NOT_SET')
assert (self.ld.resolved == 0)
def _03_same_user(self):
key = '2020-08-26T17:06:58_alejandro'
eval_str = 'PASS: Noise and artifact'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key,
user_eval=eval_str)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 1)
assert (len(traj[0]['json'][key]) == 3)
assert (traj_id != self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['extended_qc']['alignment_count'] == 1)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 0)
assert (insertion['json']['extended_qc']['experimenter'] == 'PASS')
assert (insertion['json']['qc'] == 'PASS')
assert (self.ld.resolved == 0)
def _04_two_alignments(self):
key = '2020-09-14T15:42:22_guido'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 2)
assert (len(traj[0]['json'][key]) == 2)
assert (traj_id != self.prev_traj_id)
# Also assert all the keys match
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'PASS')
assert (insertion['json']['extended_qc']['experimenter'] == 'PASS')
assert (insertion['json']['extended_qc']['alignment_count'] == 2)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 0)
assert (insertion['json']['extended_qc']['alignment_qc'] < 0.8)
assert (self.ld.resolved == 0)
def _05_three_alignments(self):
key = '2020-09-14T15:44:56_nate'
eval_str = 'WARNING: Drift'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key,
user_eval=eval_str)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (len(traj[0]['json']) == 3)
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json'][key]) == 3)
assert (traj_id != self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'WARNING')
assert (insertion['json']['extended_qc']['experimenter'] == 'WARNING')
assert (insertion['json']['extended_qc']['alignment_count'] == 3)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
def _06_new_user_after_resolved(self):
key = '2020-09-16T15:44:56_mayo'
eval_str = 'PASS: Drift'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key,
user_eval=eval_str)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (len(traj[0]['json']) == 4)
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (traj_id == self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'WARNING')
assert (insertion['json']['extended_qc']['experimenter'] == 'WARNING')
assert (insertion['json']['extended_qc']['alignment_count'] == 4)
assert (insertion['json']['extended_qc']['alignment_stored'] ==
self.resolved_key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
def _07_same_user_after_resolved(self):
key = '2020-10-14T15:44:56_nate'
eval_str = 'CRITICAL: Brain Damage'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key,
user_eval=eval_str)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 5)
assert (traj_id == self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'CRITICAL')
assert (insertion['json']['extended_qc']['experimenter'] == 'CRITICAL')
assert (insertion['json']['extended_qc']['alignment_count'] == 5)
assert (insertion['json']['extended_qc']['alignment_stored'] ==
self.resolved_key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
def _08_starting_alignments(self):
key = '2020-10-14T15:44:56_nate'
# Starting from original
self.ld.probe_id = self.probe_id2
prev_align = self.ld.get_previous_alignments()
assert (len(prev_align) == 1)
assert (prev_align[0] == 'original')
assert (self.ld.alignments == {})
# Loading in one with alignments
self.ld.probe_id = self.probe_id
prev_align = self.ld.get_previous_alignments()
assert (len(prev_align) == 6)
assert (prev_align[-1] == 'original')
assert (len(self.ld.alignments) == 5)
assert (sorted(self.ld.alignments, reverse=True)[0] == key)
# Now loading back one with nothing
self.ld.probe_id = self.probe_id2
prev_align = self.ld.get_previous_alignments()
assert (len(prev_align) == 1)
assert (prev_align[0] == 'original')
assert (self.ld.alignments == {})
@classmethod
def tearDownClass(cls) -> None:
one.alyx.rest('insertions', 'delete', id=cls.probe_id)
one.alyx.rest('insertions', 'delete', id=cls.probe_id2)
class TestsAlignmentQcGUI(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
probe = ['probe00']
create_alyx_probe_insertions(session_path=EPHYS_SESSION,
model='3B2',
labels=probe,
one=one,
force=True)
cls.probe_id = one.alyx.rest('insertions',
'list',
session=EPHYS_SESSION,
name='probe00')[0]['id']
data = np.load(Path(
Path(__file__).parent.joinpath('fixtures',
'data_alignmentqc_gui.npz')),
allow_pickle=True)
cls.xyz_picks = data['xyz_picks']
cls.alignments = data['alignments'].tolist()
cls.cluster_chns = data['cluster_chns']
register_track(cls.probe_id,
picks=cls.xyz_picks,
one=one,
overwrite=True,
channels=False)
def setUp(self) -> None:
self.resolved_key = '2020-09-14T15:44:56_nate'
self.ld = LoadData(one=one,
brain_atlas=brain_atlas,
testing=True,
probe_id=self.probe_id)
_ = self.ld.get_xyzpicks()
self.ld.cluster_chns = self.cluster_chns
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
self.ephysalign = EphysAlignment(self.ld.xyz_picks,
self.ld.chn_depths,
brain_atlas=self.ld.brain_atlas)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
if traj:
self.prev_traj_id = traj[0]['id']
def test_01_no_alignment(self):
prev_align = self.ld.get_previous_alignments()
assert (len(prev_align) == 1)
assert (prev_align[0] == 'original')
feature, track = self.ld.get_starting_alignment(0)
assert (not feature)
assert (not track)
assert (not self.ld.alignments)
assert (self.ld.resolved == 0)
def test_02_one_alignment(self):
key = '2020-07-26T17:06:58_alejandro'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 1)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['extended_qc']['alignment_count'] == 1)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (self.ld.resolved == 0)
def test_03_same_user(self):
key = '2020-08-26T17:06:58_alejandro'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 1)
assert (traj_id != self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['extended_qc']['alignment_count'] == 1)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 0)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (self.ld.resolved == 0)
def test_04_two_alignments(self):
key = '2020-09-14T15:42:22_guido'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 2)
assert (traj_id != self.prev_traj_id)
# Also assert all the keys match
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (insertion['json']['extended_qc']['alignment_count'] == 2)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 0)
assert (insertion['json']['extended_qc']['alignment_qc'] < 0.8)
assert (self.ld.resolved == 0)
def test_05_three_alignments(self):
key = '2020-09-14T15:44:56_nate'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (len(traj[0]['json']) == 3)
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (traj_id != self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (insertion['json']['extended_qc']['alignment_count'] == 3)
assert (insertion['json']['extended_qc']['alignment_stored'] == key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
def test_06_new_user_after_resolved(self):
key = '2020-09-16T15:44:56_mayo'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (len(traj[0]['json']) == 4)
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (traj_id == self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (insertion['json']['extended_qc']['alignment_count'] == 4)
assert (insertion['json']['extended_qc']['alignment_stored'] ==
self.resolved_key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
def test_07_same_user_after_resolved(self):
key = '2020-10-14T15:44:56_nate'
feature = self.alignments[key][0]
track = self.alignments[key][1]
xyz_channels = self.ephysalign.get_channel_locations(feature, track)
self.ld.upload_data(xyz_channels, channels=False)
self.ld.update_alignments(np.array(feature),
np.array(track),
key_info=key)
_ = self.ld.get_previous_alignments()
_ = self.ld.get_starting_alignment(0)
assert (self.ld.current_align == key)
traj = one.alyx.rest('trajectories',
'list',
probe_id=self.probe_id,
provenance='Ephys aligned histology track')
traj_id = traj[0]['id']
assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key)
assert (len(traj[0]['json']) == 5)
assert (traj_id == self.prev_traj_id)
self.ld.update_qc(upload_flatiron=False)
insertion = one.alyx.rest('insertions', 'read', id=self.probe_id)
assert (insertion['json']['qc'] == 'NOT_SET')
assert (insertion['json']['extended_qc']['alignment_count'] == 5)
assert (insertion['json']['extended_qc']['alignment_stored'] ==
self.resolved_key)
assert (insertion['json']['extended_qc']['alignment_resolved'] == 1)
assert (
insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc')
assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8)
assert (self.ld.resolved == 1)
@classmethod
def tearDownClass(cls) -> None:
one.alyx.rest('insertions', 'delete', id=cls.probe_id)