def get_aligned_channels(ins, chn_coords, one, ba=None, save_dir=None): ba = ba or AllenAtlas(25) depths = chn_coords[:, 1] xyz = np.array(ins['json']['xyz_picks']) / 1e6 traj = one.alyx.rest('trajectories', 'list', probe_insertion=ins['id'], provenance='Ephys aligned histology track')[0] align_key = ins['json']['extended_qc']['alignment_stored'] feature = traj['json'][align_key][0] track = traj['json'][align_key][1] ephysalign = EphysAlignment(xyz, depths, track_prev=track, feature_prev=feature, brain_atlas=ba, speedy=True) channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6) channels_atlas_ids = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id'] out_files = [] if save_dir is not None: f_name = Path(save_dir).joinpath('channels.mlapdv.npy') np.save(f_name, channels_mlapdv) out_files.append(f_name) f_name = Path(save_dir).joinpath('channels.brainLocationIds_ccf_2017.npy') np.save(f_name, channels_atlas_ids) out_files.append(f_name) return out_files
def get_channels(self, traj, ins=None, depths=None): if depths is None: depths = SITES_COORDINATES[:, 1] if traj['provenance'] == 'Planned' or traj['provenance'] == 'Micro-manipulator': ins = atlas.Insertion.from_dict(traj) # Deepest coordinate first xyz = np.c_[ins.tip, ins.entry].T xyz_channels = histology.interpolate_along_track(xyz, (depths + TIP_SIZE_UM) / 1e6) else: if ins is None: ins_idx = np.where(traj['probe_insertion'] == self.ins['ids'])[0][0] xyz = np.array(self.ins['insertions'][ins_idx]['json']['xyz_picks']) / 1e6 else: xyz = np.array(ins['json']['xyz_picks']) / 1e6 if traj['provenance'] == 'Histology track': xyz = xyz[np.argsort(xyz[:, 2]), :] xyz_channels = histology.interpolate_along_track(xyz, (depths + TIP_SIZE_UM) / 1e6) else: if ins is None: align_key = (self.ins['insertions'][ins_idx]['json']['extended_qc'] ['alignment_stored']) else: align_key = ins['json']['extended_qc']['alignment_stored'] feature = traj['json'][align_key][0] track = traj['json'][align_key][1] ephysalign = EphysAlignment(xyz, depths, track_prev=track, feature_prev=feature, brain_atlas=self.ba, speedy=True) xyz_channels = ephysalign.get_channel_locations(feature, track) return xyz_channels
def compute_similarity_matrix(self): """ Computes the similarity matrix between each alignment stored in the ephys aligned trajectory. Similarity matrix based on number of clusters that share brain region and parent brain region """ r = regions_from_allen_csv() clusters = dict() for iK, key in enumerate(self.align_keys_sorted): # Location of reference lines used for alignment feature = np.array(self.alignments[key][0]) track = np.array(self.alignments[key][1]) # Instantiate EphysAlignment object ephysalign = EphysAlignment(self.xyz_picks, self.depths, track_prev=track, feature_prev=feature, brain_atlas=self.brain_atlas) # Find xyz location of all channels xyz_channels = ephysalign.get_channel_locations(feature, track) brain_regions = ephysalign.get_brain_locations(xyz_channels) # Find the location of clusters along the alignment cluster_info = dict() cluster_info['brain_id'] = brain_regions['id'][self.cluster_chns] cluster_info['parent_id'] = r.get( ids=cluster_info['brain_id']).parent.astype(int) clusters.update({key: cluster_info}) sim_matrix = np.zeros( (len(self.align_keys_sorted), len(self.align_keys_sorted))) for ik, key in enumerate(self.align_keys_sorted): for ikk, key2 in enumerate(self.align_keys_sorted): same_id = np.where( clusters[key]['brain_id'] == clusters[key2]['brain_id'])[0] not_same_id = \ np.where(clusters[key]['brain_id'] != clusters[key2]['brain_id'])[0] same_parent = np.where( clusters[key]['parent_id'][not_same_id] == clusters[key2] ['parent_id'][not_same_id])[0] sim_matrix[ik, ikk] = len(same_id) + (len(same_parent) * 0.5) # Normalise sim_matrix_norm = sim_matrix / np.max(sim_matrix) return sim_matrix_norm
class TestsEphysReconstruction(unittest.TestCase): def setUp(self): self.ephysalign = EphysAlignment(xyz_picks, track_prev=track_prev, feature_prev=feature_prev, brain_atlas=brain_atlas) self.feature = self.ephysalign.feature_init self.track = self.ephysalign.track_init def test_channel_locations(self): xyz_channels = self.ephysalign.get_channel_locations(self.feature, self.track, depths=depths) self.assertTrue(np.all(np.isclose(xyz_channels[0, :], xyz_channels_ref[0]))) self.assertTrue(np.all(np.isclose(xyz_channels[-1, :], xyz_channels_ref[-1]))) brain_regions = self.ephysalign.get_brain_locations(xyz_channels) self.assertTrue(np.all(np.equal(np.unique(brain_regions.acronym), brain_regions_ref)))
def get_channels(self, alf_object, collection): electrodes = {} try: electrodes = self.one.load_object(self.eid, alf_object, collection=collection) electrodes['axial_um'] = electrodes['localCoordinates'][:, 1] except ALFObjectNotFound: _logger.warning(f'{alf_object} does not yet exist') if self.hist_lookup[self.histology_status] == 3: try: electrodes['atlas_id'] = electrodes['brainLocationIds_ccf_2017'] electrodes['mlapdv'] = electrodes['mlapdv'] / 1e6 except KeyError: _logger.warning('Insertion resolved but brainLocationIds_ccf_2017 attribute do not exist') if self.hist_lookup[self.histology_status] > 0 and 'atlas_id' not in electrodes.keys(): if not self.brain_atlas: self.brain_atlas = AllenAtlas() self.brain_regions = self.brain_regions or self.brain_atlas.regions if 'localCoordinates' not in electrodes.keys(): geometry = trace_header(version=1) electrodes['localCoordinates'] = np.c_[geometry['x'], geometry['y']] electrodes['axial_um'] = electrodes['localCoordinates'][:, 1] depths = electrodes['localCoordinates'][:, 1] xyz = np.array(self.ins['json']['xyz_picks']) / 1e6 if self.hist_lookup[self.histology_status] >= 2: traj = self.one.alyx.rest('trajectories', 'list', provenance='Ephys aligned histology track', probe_insertion=self.pid)[0] align_key = self.ins['json']['extended_qc']['alignment_stored'] feature = traj['json'][align_key][0] track = traj['json'][align_key][1] ephysalign = EphysAlignment(xyz, depths, track_prev=track, feature_prev=feature, brain_atlas=self.brain_atlas, speedy=True) electrodes['mlapdv'] = ephysalign.get_channel_locations(feature, track) electrodes['atlas_id'] = self.brain_atlas.regions.get(self.brain_atlas.get_labels(electrodes['mlapdv']))['id'] if self.hist_lookup[self.histology_status] == 1: xyz = xyz[np.argsort(xyz[:, 2]), :] electrodes['mlapdv'] = interpolate_along_track(xyz, (depths + TIP_SIZE_UM) / 1e6) electrodes['atlas_id'] = self.brain_atlas.regions.get(self.brain_atlas.get_labels(electrodes['mlapdv']))['id'] return electrodes
def plot_scaling(region, scale, mapper, ax): for reg, col in zip(region_scaled, scale_factor): height = np.abs(reg[1] - reg[0]) color = np.array(mapper.to_rgba(col, bytes=True)) / 255 ax.bar(x=1.1, height=height, width=0.2, color=color, bottom=reg[0], edgecolor='w') sec_ax = ax.secondary_yaxis('right') sec_ax.set_yticks(np.mean(region, axis=1)) sec_ax.set_yticklabels(np.around(scale, 2)) sec_ax.tick_params(axis="y", direction="in") sec_ax.set_ylim([20, 3840]) fig, ax = plt.subplots(1, len(alignments) + 1, figsize=(15, 15)) ephysalign = EphysAlignment(xyz_picks, depths, brain_atlas=brain_atlas) feature, track, _ = ephysalign.get_track_and_feature() channels_orig = ephysalign.get_channel_locations(feature, track) region, region_label = ephysalign.scale_histology_regions(feature, track) region_scaled, scale_factor = ephysalign.get_scale_factor(region) region_colour = ephysalign.region_colour norm = matplotlib.colors.Normalize(vmin=0.5, vmax=1.5, clip=True) mapper = matplotlib.cm.ScalarMappable(norm=norm, cmap=matplotlib.cm.seismic) ax_i = fig.axes[0] plot_regions(region, region_label, region_colour, ax_i) plot_scaling(region_scaled, scale_factor, mapper, ax_i) ax_i.set_title('Original') for iK, key in enumerate(alignments): # Location of reference lines used for alignmnet feature = np.array(alignments[key][0])
class TestsEphysAlignment(unittest.TestCase): def setUp(self): self.ephysalign = EphysAlignment(xyz_picks) self.feature = self.ephysalign.feature_init self.track = self.ephysalign.track_init def test_no_scaling(self): xyz_channels = self.ephysalign.get_channel_locations(self.feature, self.track, depths=depths) coords = np.r_[[xyz_picks[-1, :]], [xyz_channels[0, :]]] dist_to_fist_electrode = np.around(_cumulative_distance(coords)[-1], 5) assert np.isclose(dist_to_fist_electrode, (TIP_SIZE_UM + 20) / 1e6) def test_offset(self): feature_val = 500 / 1e6 track_val = 1000 / 1e6 tracks = np.sort(np.r_[self.track[[0, -1]], track_val]) track_new = self.ephysalign.feature2track(tracks, self.feature, self.track) feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val]) track_new = self.ephysalign.adjust_extremes_uniform( feature_new, track_new) xyz_channels = self.ephysalign.get_channel_locations(feature_new, track_new, depths=depths) coords = np.r_[[xyz_picks[-1, :]], [xyz_channels[0, :]]] dist_to_fist_electrode = np.around(_cumulative_distance(coords)[-1], 5) assert np.isclose(dist_to_fist_electrode, ((TIP_SIZE_UM + 20) / 1e6 + feature_val)) track_val = self.ephysalign.track2feature(track_val, feature_new, track_new) self.assertTrue(np.all(np.isclose(track_val, feature_val))) region_new, _ = self.ephysalign.scale_histology_regions( feature_new, track_new) _, scale_factor = self.ephysalign.get_scale_factor(region_new) self.assertTrue(np.all(np.isclose(scale_factor, 1))) def test_uniform_scaling(self): feature_val = np.array([500, 700, 2000]) / 1e6 track_val = np.array([1000, 1300, 2700]) / 1e6 tracks = np.sort(np.r_[self.track[[0, -1]], track_val]) track_new = self.ephysalign.feature2track(tracks, self.feature, self.track) feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val]) track_new = self.ephysalign.adjust_extremes_uniform( feature_new, track_new) region_new, _ = self.ephysalign.scale_histology_regions( feature_new, track_new) _, scale_factor = self.ephysalign.get_scale_factor(region_new) self.assertTrue(np.isclose(scale_factor[0], 1)) self.assertTrue(np.isclose(scale_factor[-1], 1)) def test_linear_scaling(self): feature_val = np.array([500, 700, 2000]) / 1e6 track_val = np.array([1000, 1300, 2700]) / 1e6 tracks = np.sort(np.r_[self.track[[0, -1]], track_val]) track_new = self.ephysalign.feature2track(tracks, self.feature, self.track) feature_new = np.sort(np.r_[self.feature[[0, -1]], feature_val]) fit = np.polyfit(feature_new[1:-1], track_new[1:-1], 1) linear_fit = np.around(1 / fit[0], 3) feature_new, track_new = self.ephysalign.adjust_extremes_linear( feature_new, track_new, extend_feature=1) region_new, _ = self.ephysalign.scale_histology_regions( feature_new, track_new) _, scale_factor = self.ephysalign.get_scale_factor(region_new) self.assertTrue(np.isclose(np.around(scale_factor[0], 3), linear_fit)) self.assertTrue(np.isclose(np.around(scale_factor[-1], 3), linear_fit))
def upload_channels(self, alignment_key, upload_alyx, upload_flatiron): """ Upload channels to alyx and flatiron based on the alignment specified by the alignment key """ feature = np.array(self.alignments[alignment_key][0]) track = np.array(self.alignments[alignment_key][1]) ephysalign = EphysAlignment(self.xyz_picks, self.depths, track_prev=track, feature_prev=feature, brain_atlas=self.brain_atlas) # Find the channels channels_mlapdv = np.int32( ephysalign.get_channel_locations(feature, track) * 1e6) channels_brainID = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id'] # Find the clusters r = regions_from_allen_csv() clusters_mlapdv = channels_mlapdv[self.cluster_chns] clusters_brainID = channels_brainID[self.cluster_chns] clusters_brainAcro = r.get(ids=clusters_brainID).acronym # upload datasets to flatiron files_to_register = [] if upload_flatiron: ftp_patcher = FTPPatcher(one=self.one) insertion = self.one.alyx.rest('insertions', 'read', id=self.eid) alf_path = self.one.path_from_eid(insertion['session']).joinpath( 'alf', insertion['name']) alf_path.mkdir(exist_ok=True, parents=True) # Make the channels.mlapdv dataset f_name = alf_path.joinpath('channels.mlapdv.npy') np.save(f_name, channels_mlapdv) files_to_register.append(f_name) # Make the channels.brainLocationIds dataset f_name = alf_path.joinpath( 'channels.brainLocationIds_ccf_2017.npy') np.save(f_name, channels_brainID) files_to_register.append(f_name) # Make the clusters.mlapdv dataset f_name = alf_path.joinpath('clusters.mlapdv.npy') np.save(f_name, clusters_mlapdv) files_to_register.append(f_name) # Make the clusters.brainLocationIds dataset f_name = alf_path.joinpath( 'clusters.brainLocationIds_ccf_2017.npy') np.save(f_name, clusters_brainID) files_to_register.append(f_name) # Make the clusters.brainLocationAcronym dataset f_name = alf_path.joinpath( 'clusters.brainLocationAcronyms_ccf_2017.npy') np.save(f_name, clusters_brainAcro) files_to_register.append(f_name) self.log.info("Writing datasets to FlatIron") ftp_patcher.create_dataset(path=files_to_register, created_by=self.one._par.ALYX_LOGIN) # Need to change channels stored on alyx as well as the stored key is not the same as the # latest key if upload_alyx: if alignment_key != self.align_keys_sorted[0]: histology.register_aligned_track(self.eid, channels_mlapdv / 1e6, chn_coords=SITES_COORDINATES, one=self.one, overwrite=True, channels=self.channels) ephys_traj = self.one.alyx.rest( 'trajectories', 'list', probe_insertion=self.eid, provenance='Ephys aligned histology track') patch_dict = {'json': self.alignments} self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'], data=patch_dict) return files_to_register
def upload_channels(self, alignment_key, upload_alyx, upload_flatiron): """ Upload channels to alyx and flatiron based on the alignment specified by the alignment key """ feature = np.array(self.alignments[alignment_key][0]) track = np.array(self.alignments[alignment_key][1]) try: meta_dset = self.one.list_datasets(self.insertion['session'], '*ap.meta', collection=f'raw_ephys_data/{self.insertion["name"]}') meta_file = self.one.load_dataset(self.insertion['session'], meta_dset[0].split('/')[-1], collection=f'raw_ephys_data/{self.insertion["name"]}', download_only=True) geometry = spikeglx.read_geometry(meta_file) chns = np.c_[geometry['x'], geometry['y']] except Exception as err: self.log.warning(f"Could not compute channel locations from meta file, errored with message: {err}. " f"Will use default Neuropixel 1 channels") geometry = trace_header(version=1) chns = np.c_[geometry['x'], geometry['y']] ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1], track_prev=track, feature_prev=feature, brain_atlas=self.brain_atlas) channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6) channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id'] # Need to change channels stored on alyx as well as the stored key is not the same as the latest key if upload_alyx: if alignment_key != self.align_keys_sorted[0]: histology.register_aligned_track(self.eid, channels_mlapdv / 1e6, chn_coords=chns, one=self.one, overwrite=True, channels=self.channels_flag, brain_atlas=self.brain_atlas) ephys_traj = self.one.alyx.get(f'/trajectories?&probe_insertion={self.eid}' '&provenance=Ephys aligned histology track', clobber=True) patch_dict = {'json': self.alignments} self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'], data=patch_dict) files_to_register = [] if upload_flatiron: ftp_patcher = FTPPatcher(one=self.one) alf_path = self.one.eid2path(self.insertion['session']).joinpath('alf', self.insertion["name"]) alf_path.mkdir(exist_ok=True, parents=True) f_name = alf_path.joinpath('electrodeSites.mlapdv.npy') np.save(f_name, channels_mlapdv) files_to_register.append(f_name) f_name = alf_path.joinpath('electrodeSites.brainLocationIds_ccf_2017.npy') np.save(f_name, channels_atlas_id) files_to_register.append(f_name) f_name = alf_path.joinpath('electrodeSites.localCoordinates.npy') np.save(f_name, chns) files_to_register.append(f_name) probe_collections = self.one.list_collections(self.insertion['session'], filename='channels*', collection=f'alf/{self.insertion["name"]}*') for collection in probe_collections: chns = self.one.load_dataset(self.insertion['session'], 'channels.localCoordinates', collection=collection) ephysalign = EphysAlignment(self.xyz_picks, chns[:, 1], track_prev=track, feature_prev=feature, brain_atlas=self.brain_atlas) channels_mlapdv = np.int32(ephysalign.get_channel_locations(feature, track) * 1e6) channels_atlas_id = ephysalign.get_brain_locations(channels_mlapdv / 1e6)['id'] alf_path = self.one.eid2path(self.insertion['session']).joinpath(collection) alf_path.mkdir(exist_ok=True, parents=True) f_name = alf_path.joinpath('channels.mlapdv.npy') np.save(f_name, channels_mlapdv) files_to_register.append(f_name) f_name = alf_path.joinpath('channels.brainLocationIds_ccf_2017.npy') np.save(f_name, channels_atlas_id) files_to_register.append(f_name) self.log.info("Writing datasets to FlatIron") ftp_patcher.create_dataset(path=files_to_register, created_by=self.one.alyx.user) return files_to_register
def plot_alignment(insertion, traj, ind=None): depths = SITES_COORDINATES[:, 1] xyz_picks = np.array(insertion['json']['xyz_picks']) / 1e6 alignments = traj['json'].copy() k = list(alignments.keys())[-1] # if only I had a Walrus available ! alignments = {k: alignments[k]} # Create a figure and arrange using gridspec widths = [1, 2.5] heights = [1] * len(alignments) gs_kw = dict(width_ratios=widths, height_ratios=heights) fig, axis = plt.subplots(len(alignments), 2, constrained_layout=True, gridspec_kw=gs_kw, figsize=(8, 9)) # Iterate over all alignments for trajectory # 1. Plot brain regions that channel pass through # 2. Plot coronal slice along trajectory with location of channels shown as red points # 3. Save results for each alignment into a dict - channels channels = {} for iK, key in enumerate(alignments): # Location of reference lines used for alignmnet feature = np.array(alignments[key][0]) track = np.array(alignments[key][1]) chn_coords = SITES_COORDINATES # Instantiate EphysAlignment object ephysalign = EphysAlignment(xyz_picks, depths, track_prev=track, feature_prev=feature) # Find xyz location of all channels xyz_channels = ephysalign.get_channel_locations(feature, track) # Find brain region that each channel is located in brain_regions = ephysalign.get_brain_locations(xyz_channels) # Add extra keys to store all useful information as one bunch object brain_regions['xyz'] = xyz_channels brain_regions['lateral'] = chn_coords[:, 0] brain_regions['axial'] = chn_coords[:, 1] # Store brain regions result in channels dict with same key as in alignment channel_info = {key: brain_regions} channels.update(channel_info) # For plotting -> extract the boundaries of the brain regions, as well as CCF label and colour region, region_label, region_colour, _ = ephysalign.get_histology_regions( xyz_channels, depths) # Make plot that shows the brain regions that channels pass through ax_regions = fig.axes[iK * 2] for reg, col in zip(region, region_colour): height = np.abs(reg[1] - reg[0]) bottom = reg[0] color = col / 255 ax_regions.bar(x=0.5, height=height, width=1, color=color, bottom=reg[0], edgecolor='w') ax_regions.set_yticks(region_label[:, 0].astype(int)) ax_regions.yaxis.set_tick_params(labelsize=8) ax_regions.get_xaxis().set_visible(False) ax_regions.set_yticklabels(region_label[:, 1]) ax_regions.spines['right'].set_visible(False) ax_regions.spines['top'].set_visible(False) ax_regions.spines['bottom'].set_visible(False) ax_regions.hlines([0, 3840], *ax_regions.get_xlim(), linestyles='dashed', linewidth=3, colors='k') # ax_regions.plot(np.ones(channel_depths_track.shape), channel_depths_track, '*r') # Make plot that shows coronal slice that trajectory passes through with location of channels # shown in red ax_slice = fig.axes[iK * 2 + 1] brain_atlas.plot_tilted_slice(xyz_channels, axis=1, ax=ax_slice) ax_slice.plot(xyz_channels[:, 0] * 1e6, xyz_channels[:, 2] * 1e6, 'r*') ax_slice.title.set_text(insertion['id'] + '\n' + str(key)) # Make sure the plot displays plt.show()
class TestsAlignmentQcGUI(unittest.TestCase): @classmethod def setUpClass(cls) -> None: probe = ['probe00', 'probe01'] create_alyx_probe_insertions(session_path=EPHYS_SESSION, model='3B2', labels=probe, one=one, force=True) cls.probe_id = one.alyx.rest('insertions', 'list', session=EPHYS_SESSION, name='probe00')[0]['id'] cls.probe_id2 = one.alyx.rest('insertions', 'list', session=EPHYS_SESSION, name='probe01')[0]['id'] data = np.load(Path( Path(__file__).parent.joinpath('fixtures', 'data_alignmentqc_gui.npz')), allow_pickle=True) cls.xyz_picks = data['xyz_picks'] cls.alignments = data['alignments'].tolist() cls.cluster_chns = data['cluster_chns'] register_track(cls.probe_id, picks=cls.xyz_picks, one=one, overwrite=True, channels=False) register_track(cls.probe_id2, picks=cls.xyz_picks, one=one, overwrite=True, channels=False) def setUp(self) -> None: self.resolved_key = '2020-09-14T15:44:56_nate' self.ld = LoadData(one=one, brain_atlas=brain_atlas, testing=True, probe_id=self.probe_id) _ = self.ld.get_xyzpicks() self.ld.cluster_chns = self.cluster_chns _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) self.ephysalign = EphysAlignment(self.ld.xyz_picks, self.ld.chn_depths, brain_atlas=self.ld.brain_atlas) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') if traj: self.prev_traj_id = traj[0]['id'] def test_alignments(self): checks = getmembers( TestsAlignmentQcGUI, lambda x: isfunction(x) and re.match(r'^_\d{2}_.*', x.__name__)) # Run each function in order for name, fn in sorted(checks, key=lambda x: x[0]): if not name.startswith('_01_'): self.setUp() fn(self) def _01_no_alignment(self): prev_align = self.ld.get_previous_alignments() assert (len(prev_align) == 1) assert (prev_align[0] == 'original') feature, track = self.ld.get_starting_alignment(0) assert (not feature) assert (not track) assert (not self.ld.alignments) assert (self.ld.resolved == 0) def _02_one_alignment(self): key = '2020-07-26T17:06:58_alejandro' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 1) # Not added user evaluation so should just contain feature and track lists assert (len(traj[0]['json'][key]) == 2) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['extended_qc']['alignment_count'] == 1) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (not insertion['json']['extended_qc'].get('experimenter', None)) assert (insertion['json']['qc'] == 'NOT_SET') assert (self.ld.resolved == 0) def _03_same_user(self): key = '2020-08-26T17:06:58_alejandro' eval_str = 'PASS: Noise and artifact' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key, user_eval=eval_str) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 1) assert (len(traj[0]['json'][key]) == 3) assert (traj_id != self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['extended_qc']['alignment_count'] == 1) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 0) assert (insertion['json']['extended_qc']['experimenter'] == 'PASS') assert (insertion['json']['qc'] == 'PASS') assert (self.ld.resolved == 0) def _04_two_alignments(self): key = '2020-09-14T15:42:22_guido' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 2) assert (len(traj[0]['json'][key]) == 2) assert (traj_id != self.prev_traj_id) # Also assert all the keys match self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'PASS') assert (insertion['json']['extended_qc']['experimenter'] == 'PASS') assert (insertion['json']['extended_qc']['alignment_count'] == 2) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 0) assert (insertion['json']['extended_qc']['alignment_qc'] < 0.8) assert (self.ld.resolved == 0) def _05_three_alignments(self): key = '2020-09-14T15:44:56_nate' eval_str = 'WARNING: Drift' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key, user_eval=eval_str) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (len(traj[0]['json']) == 3) assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json'][key]) == 3) assert (traj_id != self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'WARNING') assert (insertion['json']['extended_qc']['experimenter'] == 'WARNING') assert (insertion['json']['extended_qc']['alignment_count'] == 3) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) def _06_new_user_after_resolved(self): key = '2020-09-16T15:44:56_mayo' eval_str = 'PASS: Drift' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key, user_eval=eval_str) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (len(traj[0]['json']) == 4) assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (traj_id == self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'WARNING') assert (insertion['json']['extended_qc']['experimenter'] == 'WARNING') assert (insertion['json']['extended_qc']['alignment_count'] == 4) assert (insertion['json']['extended_qc']['alignment_stored'] == self.resolved_key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) def _07_same_user_after_resolved(self): key = '2020-10-14T15:44:56_nate' eval_str = 'CRITICAL: Brain Damage' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key, user_eval=eval_str) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 5) assert (traj_id == self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'CRITICAL') assert (insertion['json']['extended_qc']['experimenter'] == 'CRITICAL') assert (insertion['json']['extended_qc']['alignment_count'] == 5) assert (insertion['json']['extended_qc']['alignment_stored'] == self.resolved_key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) def _08_starting_alignments(self): key = '2020-10-14T15:44:56_nate' # Starting from original self.ld.probe_id = self.probe_id2 prev_align = self.ld.get_previous_alignments() assert (len(prev_align) == 1) assert (prev_align[0] == 'original') assert (self.ld.alignments == {}) # Loading in one with alignments self.ld.probe_id = self.probe_id prev_align = self.ld.get_previous_alignments() assert (len(prev_align) == 6) assert (prev_align[-1] == 'original') assert (len(self.ld.alignments) == 5) assert (sorted(self.ld.alignments, reverse=True)[0] == key) # Now loading back one with nothing self.ld.probe_id = self.probe_id2 prev_align = self.ld.get_previous_alignments() assert (len(prev_align) == 1) assert (prev_align[0] == 'original') assert (self.ld.alignments == {}) @classmethod def tearDownClass(cls) -> None: one.alyx.rest('insertions', 'delete', id=cls.probe_id) one.alyx.rest('insertions', 'delete', id=cls.probe_id2)
class TestsAlignmentQcGUI(unittest.TestCase): @classmethod def setUpClass(cls) -> None: probe = ['probe00'] create_alyx_probe_insertions(session_path=EPHYS_SESSION, model='3B2', labels=probe, one=one, force=True) cls.probe_id = one.alyx.rest('insertions', 'list', session=EPHYS_SESSION, name='probe00')[0]['id'] data = np.load(Path( Path(__file__).parent.joinpath('fixtures', 'data_alignmentqc_gui.npz')), allow_pickle=True) cls.xyz_picks = data['xyz_picks'] cls.alignments = data['alignments'].tolist() cls.cluster_chns = data['cluster_chns'] register_track(cls.probe_id, picks=cls.xyz_picks, one=one, overwrite=True, channels=False) def setUp(self) -> None: self.resolved_key = '2020-09-14T15:44:56_nate' self.ld = LoadData(one=one, brain_atlas=brain_atlas, testing=True, probe_id=self.probe_id) _ = self.ld.get_xyzpicks() self.ld.cluster_chns = self.cluster_chns _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) self.ephysalign = EphysAlignment(self.ld.xyz_picks, self.ld.chn_depths, brain_atlas=self.ld.brain_atlas) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') if traj: self.prev_traj_id = traj[0]['id'] def test_01_no_alignment(self): prev_align = self.ld.get_previous_alignments() assert (len(prev_align) == 1) assert (prev_align[0] == 'original') feature, track = self.ld.get_starting_alignment(0) assert (not feature) assert (not track) assert (not self.ld.alignments) assert (self.ld.resolved == 0) def test_02_one_alignment(self): key = '2020-07-26T17:06:58_alejandro' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 1) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['extended_qc']['alignment_count'] == 1) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['qc'] == 'NOT_SET') assert (self.ld.resolved == 0) def test_03_same_user(self): key = '2020-08-26T17:06:58_alejandro' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 1) assert (traj_id != self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['extended_qc']['alignment_count'] == 1) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 0) assert (insertion['json']['qc'] == 'NOT_SET') assert (self.ld.resolved == 0) def test_04_two_alignments(self): key = '2020-09-14T15:42:22_guido' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 2) assert (traj_id != self.prev_traj_id) # Also assert all the keys match self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'NOT_SET') assert (insertion['json']['extended_qc']['alignment_count'] == 2) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 0) assert (insertion['json']['extended_qc']['alignment_qc'] < 0.8) assert (self.ld.resolved == 0) def test_05_three_alignments(self): key = '2020-09-14T15:44:56_nate' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (len(traj[0]['json']) == 3) assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (traj_id != self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'NOT_SET') assert (insertion['json']['extended_qc']['alignment_count'] == 3) assert (insertion['json']['extended_qc']['alignment_stored'] == key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) def test_06_new_user_after_resolved(self): key = '2020-09-16T15:44:56_mayo' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (len(traj[0]['json']) == 4) assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (traj_id == self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'NOT_SET') assert (insertion['json']['extended_qc']['alignment_count'] == 4) assert (insertion['json']['extended_qc']['alignment_stored'] == self.resolved_key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) def test_07_same_user_after_resolved(self): key = '2020-10-14T15:44:56_nate' feature = self.alignments[key][0] track = self.alignments[key][1] xyz_channels = self.ephysalign.get_channel_locations(feature, track) self.ld.upload_data(xyz_channels, channels=False) self.ld.update_alignments(np.array(feature), np.array(track), key_info=key) _ = self.ld.get_previous_alignments() _ = self.ld.get_starting_alignment(0) assert (self.ld.current_align == key) traj = one.alyx.rest('trajectories', 'list', probe_id=self.probe_id, provenance='Ephys aligned histology track') traj_id = traj[0]['id'] assert (sorted(list(traj[0]['json'].keys()), reverse=True)[0] == key) assert (len(traj[0]['json']) == 5) assert (traj_id == self.prev_traj_id) self.ld.update_qc(upload_flatiron=False) insertion = one.alyx.rest('insertions', 'read', id=self.probe_id) assert (insertion['json']['qc'] == 'NOT_SET') assert (insertion['json']['extended_qc']['alignment_count'] == 5) assert (insertion['json']['extended_qc']['alignment_stored'] == self.resolved_key) assert (insertion['json']['extended_qc']['alignment_resolved'] == 1) assert ( insertion['json']['extended_qc']['alignment_resolved_by'] == 'qc') assert (insertion['json']['extended_qc']['alignment_qc'] > 0.8) assert (self.ld.resolved == 1) @classmethod def tearDownClass(cls) -> None: one.alyx.rest('insertions', 'delete', id=cls.probe_id)