def gen_clusters_subselection(clusters_subselection_all, code): kwik_file = join(basic_dir, geometry_dir.format(channel_number[code], geometry_descriptions[code]), 'threshold_6_5std.kwik') h5file = h5.File(kwik_file, mode='r') extra_spike_times_subselection = np.array(list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() clusters_subselection = {} for c in np.arange(len(clusters)): common_spikes, indices_of_common_spikes, not_common_spikes = \ ut.find_points_in_array_with_jitter(extra_spike_times_128ch[clusters[c]], extra_spike_times_subselection, 7) clusters_subselection[c] = indices_of_common_spikes clusters_subselection_all[code] = clusters_subselection return clusters_subselection_all
def gen_clusters_subselection(clusters_subselection_all, code): kwik_file = join( basic_dir, geometry_dir.format(channel_number[code], geometry_descriptions[code]), 'threshold_6_5std.kwik') h5file = h5.File(kwik_file, mode='r') extra_spike_times_subselection = np.array( list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() clusters_subselection = {} for c in np.arange(len(clusters)): common_spikes, indices_of_common_spikes, not_common_spikes = \ ut.find_points_in_array_with_jitter(extra_spike_times_128ch[clusters[c]], extra_spike_times_subselection, 7) clusters_subselection[c] = indices_of_common_spikes clusters_subselection_all[code] = clusters_subselection return clusters_subselection_all
def create_juxta_label(kwik_file, spike_thresholds, num_of_spike_groups=1, adc_channel_used=0, adc_dtype=np.uint16, inter_spike_time_distance=0.002, amp_gain=100, num_of_raw_data_channels=None, spike_channels=None, verbose=True): """ Find the juxta spikes in the extra spike train and label them according to size splitting them into num_of_spike_groups groups Parameters ---------- kwik_file spike_thresholds num_of_spike_groups adc_channel_used adc_dtype inter_spike_time_distance amp_gain num_of_raw_data_channels spike_channels verbose Returns ------- """ h5file = h5.File(kwik_file, mode='r') extra_spike_times = np.array( list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() spikes_used = len(extra_spike_times) if verbose: print("Total spikes in extra = " + str(len(extra_spike_times))) # Get the juxta spikes and generate labels # 1) Generate the juxta spike time triggers (and the adc traces in Volts) raw_juxta_data_file = r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Data' + \ r'\adc2015-09-03T21_18_47.bin' raw_data_patch = ephys.load_raw_event_trace(raw_juxta_data_file, number_of_channels=8, channel_used=adc_channel_used, dtype=adc_dtype) juxta_spike_triggers, juxta_spike_peaks, juxta_spike_data_in_V = tf.create_spike_triggered_events( raw_data_patch.dataMatrix, threshold=spike_thresholds, inter_spike_time_distance=inter_spike_time_distance, amp_gain=amp_gain) num_of_spikes = len(juxta_spike_triggers) if verbose: print('Total spikes in Juxta = ' + str(num_of_spikes)) # 2) Seperate the juxta spikes into a number of groups according to their size juxta_spikes_grouped, juxta_spike_peaks_grouped, juxta_spike_triggers_grouped_withnans,\ juxta_spike_peaks_grouped_withnans, spike_thresholds_groups = \ split_juxta_spikes_into_groups_by_size(num_of_spike_groups=num_of_spike_groups, juxta_spike_peaks=juxta_spike_peaks, juxta_spike_triggers=juxta_spike_triggers) # 3) Find the common spikes between the extra apikes and the juxta spikes for all the juxta spikes and for all the # sub groups of juxta spikes and group the good spikes common_spikes_grouped = {} juxta_spikes_not_found_grouped = {} indices_of_common_extra_spikes_grouped = {} for g in range(1, num_of_spike_groups + 1): common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g], juxta_spikes_not_found_grouped[g] = \ ut.find_points_in_array_with_jitter(array_of_points_to_be_found=juxta_spikes_grouped[g], array_to_search=extra_spike_times[:spikes_used], jitter_around_each_point=7) if spike_channels is not None and num_of_raw_data_channels is not None: common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g] \ = select_spikes_in_certain_channels(spike_thresholds, common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g], spike_channels, num_of_raw_data_channels) # 5) Get the t-sne indices of the grouped juxta spikes indices_of_data_for_tsne = range(spikes_used) juxta_cluster_indices_grouped = {} for g in range(0, num_of_spike_groups): juxta_cluster_indices_temp = np.intersect1d( indices_of_data_for_tsne, indices_of_common_extra_spikes_grouped[g + 1]) juxta_cluster_indices_grouped[g] = [ i for i in np.arange(0, len(indices_of_data_for_tsne)) if len( np.where(juxta_cluster_indices_temp == indices_of_data_for_tsne[i])[0]) ] if verbose and spike_channels is not None: print('Labeled after cleaning = ' + str(len(juxta_cluster_indices_grouped[g]))) return juxta_cluster_indices_grouped, spike_thresholds_groups
h5file = h5.File(kwik_file_path, mode='r') spike_times = np.array(list(h5file['channel_groups/1/spikes/time_samples'])) h5file.close() spikes_used = 400000#len(spike_times)#130000 # Get clusters mat_dict = sio.loadmat(spike_mat_file) labeled_spike_times = mat_dict['gtTimes'][0] # 1) Get indices of labeled spikes spikes_labeled_dict = dict() number_of_labels = labeled_spike_times.__len__() for i in range(number_of_labels): common_spikes, spikes_labeled_dict[i], labeled_spikes_not_found = \ ut.find_points_in_array_with_jitter(labeled_spike_times[i][:, 0], spike_times[:spikes_used], 6) # 2) Generate a labels array (each spike is represented by its label number or -1 if it doesn't have a label labels = np.zeros(spikes_used) labels[0:] = -1 for l in range(number_of_labels): labels[spikes_labeled_dict[l]] = l # 3) Find how many spikes are labeled number_of_labeled_spikes = 0 for i in range(number_of_labels): number_of_labeled_spikes += labeled_spike_times[i][:, 0].shape[0]
spike_times_phy = np.array(list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() t_tsne = np.load(r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\klustakwik'+\ r'\threshold_6_5std\t_sne_results_final_allspikes.npy') kilosort_experiment_folder = r'thres4_10_10_Fe16_Pc12' # thres4_10_10_Fe16_Pc12 OR thres4_10_10_Fe256_Pc128 OR thres6_12_12_Fe256_Pc128 kilosort_path = os.path.join(r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\kilosort', kilosort_experiment_folder) spike_clusters_kilosort = np.load(os.path.join(kilosort_path, 'kilosort output\\spike_templates.npy')) spike_times_kilosort = np.load(os.path.join(kilosort_path, 'kilosort output\spike_times.npy')) template_features = np.load(os.path.join(kilosort_path, 'kilosort output\\template_features.npy')) template_features_ind = np.load(os.path.join(kilosort_path, 'kilosort output\\template_feature_ind.npy')) pc_features = np.load(os.path.join(kilosort_path, 'kilosort output\\pc_features.npy')) pc_features_ind = np.load(os.path.join(kilosort_path, 'kilosort output\\pc_feature_ind.npy')) common_spikes, indices_of_common_spikes_in_phy, indices_of_common_spikes_in_kilosort, small_train_spikes_not_found = util.find_points_in_array_with_jitter(spike_times_phy, spike_times_kilosort, 6) np.save(os.path.join(kilosort_path, 'tsne\\small_train_spikes_not_found.npy'), small_train_spikes_not_found) np.save(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes.npy'), indices_of_common_spikes_in_phy) np.save(os.path.join(kilosort_path, 'tsne\common_spikes_in_tsne_train.npy'), common_spikes) np.save(os.path.join(kilosort_path, 'tsne\indices_of_common_spikes_in_kilosort_train.npy'), indices_of_common_spikes_in_kilosort) # OR common_spikes = np.load(os.path.join(kilosort_path, 'tsne\\common_spikes_in_tsne_train.npy')) indices_of_common_spikes_in_phy = np.load(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes.npy')) indices_of_common_spikes_in_kilosort = np.load(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes_in_kilosort_train.npy')) small_train_spikes_not_found = np.load(os.path.join(kilosort_path, 'tsne\\small_train_spikes_not_found.npy')) # plot the phy tsne using the clusters defined by kilosort kilosort_units = {} # dict has all the phy spikes for i in np.arange(indices_of_common_spikes_in_kilosort.__len__()):
spike_channels = np.argmin(t, axis=0) good_spike_indices = [ i for i, x in list(enumerate(spike_channels)) if np.in1d(x, good_channels) ] common_spikes = common_spikes[good_spike_indices] indices_of_common_spikes_in_extra = indices_of_common_spikes_in_extra[ good_spike_indices] return common_spikes, indices_of_common_spikes_in_extra # 3) Find the common spikes between the extra apikes and the juxta spikes for all the juxta spikes and for all the # sub groups of juxta spikes common_spikes, indices_of_common_spikes_in_extra, juxta_spikes_not_found = \ ut.find_points_in_array_with_jitter(array_of_points_to_be_found=juxta_spike_triggers, array_to_search=extra_spike_times[:spikes_used], jitter_around_each_point=7) common_spikes_chan_selected = select_spikes_in_certain_channels( common_spikes, indices_of_common_spikes_in_extra, raw_data, good_channels) # 4) Group the good spikes common_spikes_grouped = {} common_spikes_grouped_chan_selected = {} juxta_spikes_not_found_grouped = {} indices_of_common_extra_spikes_grouped = {} indices_of_common_extra_spikes_grouped_chan_selected = {} for g in range(1, num_of_spike_groups + 1): common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g], juxta_spikes_not_found_grouped[g] = \ ut.find_points_in_array_with_jitter(array_of_points_to_be_found=juxta_spikes_grouped[g], array_to_search=extra_spike_times[:spikes_used], jitter_around_each_point=7)
def create_juxta_label(kwik_file, spike_thresholds, num_of_spike_groups=1, adc_channel_used=0, adc_dtype=np.uint16, inter_spike_time_distance=0.002, amp_gain=100, num_of_raw_data_channels=None, spike_channels=None, verbose=True): """ Find the juxta spikes in the extra spike train and label them according to size splitting them into num_of_spike_groups groups Parameters ---------- kwik_file spike_thresholds num_of_spike_groups adc_channel_used adc_dtype inter_spike_time_distance amp_gain num_of_raw_data_channels spike_channels verbose Returns ------- """ h5file = h5.File(kwik_file, mode='r') extra_spike_times = np.array(list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() spikes_used = len(extra_spike_times) if verbose: print("Total spikes in extra = " + str(len(extra_spike_times))) # Get the juxta spikes and generate labels # 1) Generate the juxta spike time triggers (and the adc traces in Volts) raw_juxta_data_file = r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Data' + \ r'\adc2015-09-03T21_18_47.bin' raw_data_patch = ephys.load_raw_event_trace(raw_juxta_data_file, number_of_channels=8, channel_used=adc_channel_used, dtype=adc_dtype) juxta_spike_triggers, juxta_spike_peaks, juxta_spike_data_in_V = tf.create_spike_triggered_events(raw_data_patch.dataMatrix, threshold=spike_thresholds, inter_spike_time_distance=inter_spike_time_distance, amp_gain=amp_gain) num_of_spikes = len(juxta_spike_triggers) if verbose: print('Total spikes in Juxta = ' + str(num_of_spikes)) # 2) Seperate the juxta spikes into a number of groups according to their size juxta_spikes_grouped, juxta_spike_peaks_grouped, juxta_spike_triggers_grouped_withnans,\ juxta_spike_peaks_grouped_withnans, spike_thresholds_groups = \ split_juxta_spikes_into_groups_by_size(num_of_spike_groups=num_of_spike_groups, juxta_spike_peaks=juxta_spike_peaks, juxta_spike_triggers=juxta_spike_triggers) # 3) Find the common spikes between the extra apikes and the juxta spikes for all the juxta spikes and for all the # sub groups of juxta spikes and group the good spikes common_spikes_grouped = {} juxta_spikes_not_found_grouped = {} indices_of_common_extra_spikes_grouped = {} for g in range(1, num_of_spike_groups+1): common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g], juxta_spikes_not_found_grouped[g] = \ ut.find_points_in_array_with_jitter(array_of_points_to_be_found=juxta_spikes_grouped[g], array_to_search=extra_spike_times[:spikes_used], jitter_around_each_point=7) if spike_channels is not None and num_of_raw_data_channels is not None: common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g] \ = select_spikes_in_certain_channels(spike_thresholds, common_spikes_grouped[g], indices_of_common_extra_spikes_grouped[g], spike_channels, num_of_raw_data_channels) # 5) Get the t-sne indices of the grouped juxta spikes indices_of_data_for_tsne = range(spikes_used) juxta_cluster_indices_grouped = {} for g in range(0, num_of_spike_groups): juxta_cluster_indices_temp = np.intersect1d(indices_of_data_for_tsne, indices_of_common_extra_spikes_grouped[g+1]) juxta_cluster_indices_grouped[g] = [i for i in np.arange(0, len(indices_of_data_for_tsne)) if len(np.where(juxta_cluster_indices_temp == indices_of_data_for_tsne[i])[0])] if verbose and spike_channels is not None: print('Labeled after cleaning = ' + str(len(juxta_cluster_indices_grouped[g]))) return juxta_cluster_indices_grouped, spike_thresholds_groups
spike_times_phy = np.array(list(h5file['channel_groups/0/spikes/time_samples'])) h5file.close() t_tsne = np.load(r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\klustakwik'+\ r'\threshold_6_5std\t_sne_results_final_allspikes.npy') kilosort_experiment_folder = r'thres4_10_10_Fe16_Pc12' # thres4_10_10_Fe16_Pc12 OR thres4_10_10_Fe256_Pc128 OR thres6_12_12_Fe256_Pc128 kilosort_path = os.path.join(r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\kilosort', kilosort_experiment_folder) spike_clusters_kilosort = np.load(os.path.join(kilosort_path, 'kilosort output\\spike_templates.npy')) spike_times_kilosort = np.load(os.path.join(kilosort_path, 'kilosort output\spike_times.npy')) template_features = np.load(os.path.join(kilosort_path, 'kilosort output\\template_features.npy')) template_features_ind = np.load(os.path.join(kilosort_path, 'kilosort output\\template_feature_ind.npy')) pc_features = np.load(os.path.join(kilosort_path, 'kilosort output\\pc_features.npy')) pc_features_ind = np.load(os.path.join(kilosort_path, 'kilosort output\\pc_feature_ind.npy')) common_spikes, indices_of_common_spikes_in_phy, indices_of_common_spikes_in_kilosort, small_train_spikes_not_found = util.find_points_in_array_with_jitter(spike_times_phy, spike_times_kilosort, 6) np.save(os.path.join(kilosort_path, 'tsne\\small_train_spikes_not_found.npy'), small_train_spikes_not_found) np.save(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes.npy'), indices_of_common_spikes_in_phy) np.save(os.path.join(kilosort_path, 'tsne\common_spikes_in_tsne_train.npy'), common_spikes) np.save(os.path.join(kilosort_path, 'tsne\indices_of_common_spikes_in_kilosort_train.npy'), indices_of_common_spikes_in_kilosort) # OR common_spikes = np.load(os.path.join(kilosort_path, 'tsne\\common_spikes_in_tsne_train.npy')) indices_of_common_spikes_in_phy = np.load(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes.npy')) indices_of_common_spikes_in_kilosort = np.load(os.path.join(kilosort_path, 'tsne\\indices_of_common_spikes_in_kilosort_train.npy')) small_train_spikes_not_found = np.load(os.path.join(kilosort_path, 'tsne\\small_train_spikes_not_found.npy')) # plot the phy tsne using the clusters defined by kilosort kilosort_units = {} # dict has all the phy spikes for i in np.arange(indices_of_common_spikes_in_kilosort.__len__()):
h5file = h5.File(kwik_file_path, mode='r') spike_times = np.array(list(h5file['channel_groups/1/spikes/time_samples'])) h5file.close() spikes_used = 400000 #len(spike_times)#130000 # Get clusters mat_dict = sio.loadmat(spike_mat_file) labeled_spike_times = mat_dict['gtTimes'][0] # 1) Get indices of labeled spikes spikes_labeled_dict = dict() number_of_labels = labeled_spike_times.__len__() for i in range(number_of_labels): common_spikes, spikes_labeled_dict[i], labeled_spikes_not_found = \ ut.find_points_in_array_with_jitter(labeled_spike_times[i][:, 0], spike_times[:spikes_used], 6) # 2) Generate a labels array (each spike is represented by its label number or -1 if it doesn't have a label labels = np.zeros(spikes_used) labels[0:] = -1 for l in range(number_of_labels): labels[spikes_labeled_dict[l]] = l # 3) Find how many spikes are labeled number_of_labeled_spikes = 0 for i in range(number_of_labels): number_of_labeled_spikes += labeled_spike_times[i][:, 0].shape[0] # Run t-sne path_to_save_tmp_data = tsne_video_path perplexity = 200