def make_video_of_tsne_iterations(iterations, video_dir, data_file_name='interim_{:0>6}.dat',
video_file_name='tsne_video.mp4', figsize=(15, 15), dpi=200, fps=30,
movie_metadata=None, labels_dict=None, cm=None, cm_remapping=None, subtitle=None,
label_name='Label', legent_on=True, label_array=None, labeled_sizes=None,
unlabeled_sizes=None, markers=None, color=None, max_screen=False):
iters = np.arange(iterations)
FFMpegWriter = animation.writers['ffmpeg']
metadata = None
if movie_metadata:
metadata = movie_metadata
writer = FFMpegWriter(fps=fps, bitrate=-1, metadata=metadata)
if cm is None:
cm = plt.cm.Dark2
fig = plt.figure(figsize=figsize, dpi=dpi)
ax = fig.add_subplot(111)
fig.tight_layout()
with writer.saving(fig, join(video_dir, video_file_name), dpi):
for it in iters:
ax.cla()
tsne = io.load_tsne_result(video_dir, data_file_name.format(it))
tsne = np.transpose(tsne)
plot_tsne(tsne, labels_dict=labels_dict, cm=cm, cm_remapping=cm_remapping, subtitle=subtitle,
label_name=label_name, legent_on=legent_on, label_array=label_array, axes=ax,
unlabeled_sizes=unlabeled_sizes, labeled_sizes=labeled_sizes, markers=markers, color=color,
max_screen=max_screen, hide_ticklabels=True)
min_x = np.min(tsne[0, :])
max_x = np.max(tsne[0, :])
min_y = np.min(tsne[1, :])
max_y = np.max(tsne[1, :])
range_x = np.max(np.abs([min_x, max_x]))
range_y = np.max(np.abs([min_y, max_y]))
plt.ylim([-range_y, range_y])
plt.xlim([-range_x, range_x])
writer.grab_frame()
if it%100 == 0:
print('Done '+str(it) + ' frames')
verbose=verbose)
tsne_results = tsne.t_sne_from_existing_distances(files_dir=tsne_cortex_folder,
data_has_exageration=True,
num_dims=num_dims,
theta=theta,
iterations=iterations,
random_seed=random_seed,
verbose=verbose,
exe_dir=exe_dir)
spike_info = preproc_kilo.generate_spike_info_from_full_tsne(
kilosort_folder=kilosort_folder_denoised, tsne_folder=tsne_cortex_folder)
# OR Load previously run t-sne
tsne_results = tsne_io.load_tsne_result(files_dir=tsne_cortex_folder)
# and previously generated spike_info
spike_info = pd.read_pickle(join(tsne_cortex_folder, 'spike_info.df'))
# Have a look
viz.plot_tsne_of_spikes(spike_info=spike_info, legent_on=False)
# Update the original spike info (created after just cleaning) with the new spike_info information from manually sorting
# on the t-sne
spike_info_after_cleaning = preproc_kilo.generate_spike_info_after_cleaning(
kilosort_folder_denoised)
spike_info_cortex_sorted = spike_info
tsne_filename = join(tsne_cortex_folder, 'result.dat')
spike_info_after_cortex_sorting_file = join(
kilosort_folder, 'spike_info_after_cortex_sorting.df')
number_of_top_pcs = 40
num_dims = 2
perplexity = 100
theta = 0.3
eta = 200
exageration = 12
iterations = 4000
random_seed = 1
verbose = 1
tsne_spike_rates_binary_pcs_0p1 = tsne.t_sne(pcs_ar_bin_0p1[:, :number_of_top_pcs], tsne_folder, barnes_hut_exe_dir,
num_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, exageration=exageration,
iterations=iterations, random_seed=random_seed, verbose=verbose)
tsne_result = tsne_io.load_tsne_result(tsne_folder)
plt.scatter(tsne_result[:, 0], tsne_result[:, 1], s=5, c='k')
tsne_pcs_count_image_file = join(tsne_folder, 'Tsne_of_13_top_PCs_of_0p1_count_spike_vectors.png')
tsne_pcs_count_image = plt.imread(tsne_pcs_count_image_file)
tsne_pcs_count_image_extent = [-4, 4, -4.6, 4.8]
# </editor-fold>
# -------------------------------------------------
# -------------------------------------------------
# <editor-fold desc="SHOW TSNE CORRELATED WITH VIDEO TSNE AND COMPARE TO FULL PROBE EQUIVALENT">
tsne_pcs_count_image_file = join(tsne_folder, 'Tsne_of_40_top_PCs_of_0p1_count_spike_vectors.png')
tsne_pcs_count_image = plt.imread(tsne_pcs_count_image_file)
tsne_pcs_count_image_extent = [-4, 4, -4.6, 4.8]
random_seed = 1
verbose = 2
tsne_spike_rates_binary_pcs_0p1 = tsne.t_sne(
pcs_ar_bin_0p1[:, :number_of_top_pcs],
tsne_folder,
barnes_hut_exe_dir,
num_dims=num_dims,
perplexity=perplexity,
theta=theta,
eta=eta,
exageration=exageration,
iterations=iterations,
random_seed=random_seed,
verbose=verbose)
tsne_result = tsne_io.load_tsne_result(tsne_folder)
# </editor-fold>
# -------------------------------------------------
# -------------------------------------------------
# <editor-fold desc="CREATE MATRIX OF COUNTS PER 100ms, PCA THIS AND RUN T-SNE">
tsne_folder = join(tsne_folder_base,
'All_spikes_100msbin_count_top500PCs_4Kiter')
time_to_bin = 0.1
frames_to_bin = time_to_bin * 120
spike_rates_count_0p1 = binning.rolling_window_with_step(
spike_rates_per_video_frame, np.sum, frames_to_bin, frames_to_bin)
spike_rates_count_0p1 = spike_rates_count_0p1.transpose() * 0.00833
num_dims=num_dims,
perplexity=perplexity,
theta=theta,
eta=eta,
exageration=exageration,
iterations=iterations,
random_seed=random_seed,
verbose=verbose)
# </editor-fold>
# -------------------------------------------------
# -------------------------------------------------
# <editor-fold desc="DBSCAN THE TSNE AND SAVE THE RESULTING LABELS">
tsne_result = tsne_io.load_tsne_result(tsne_folder)
X = tsne_result.transpose()
db, n_clusters_, labels, core_samples_mask, score = tsne_funcs.fit_dbscan(
X, eps=0.015, min_samples=40, normalize=True, show=True)
pickle.dump(db, open(join(tsne_folder, 'dbscan_result.pcl'), 'wb'))
np.save(join(tsne_folder, 'dbscan_labels.npy'), db.labels_)
# </editor-fold>
# -------------------------------------------------
# -------------------------------------------------
# <editor-fold desc="SHOW THE FRAMES AND THE PCs ON THE CLUSTERED T-SNE">
tsne_result = tsne_io.load_tsne_result(tsne_folder)
db = pickle.load(open(join(tsne_folder, 'dbscan_result.pcl'), 'rb'))
iterations = 4000
random_seed = 1
verbose = 3
tsne_results = tsne.t_sne_from_existing_distances(files_dir=tsne_folder,
data_has_exageration=True,
num_dims=num_dims,
theta=theta,
iterations=iterations,
random_seed=random_seed,
verbose=verbose)
spike_info = preproc_kilo.generate_spike_info_from_full_tsne(
kilosort_folder=kilosort_folder, tsne_folder=tsne_folder)
# OR Load previously run t-sne
tsne_results = tsne_io.load_tsne_result(files_dir=tsne_folder)
# and previously generated spike_info
spike_info = pd.read_pickle(join(tsne_folder, 'spike_info.df'))
# Have a look
viz.plot_tsne_of_spikes(spike_info=spike_info, legent_on=False)
# ------------------------------------------
# Now run the T-Sne Gui to manual sort
# ------------------------------------------
# ------------------------------------------
# Get the positions of the templates and have a look
sp_pos.view_grouped_templates_positions(kilosort_folder, const.BRAIN_REGIONS,
const.PROBE_DIMENSIONS,
r'\All_spikes_100msbin_count_top40PCs_6Kiter'
}
video_tsne_folders = {'33.1': r'Y:\swc\kampff\George\DataAndResults\Experiments\Awake\NeuroSeeker\AK_33.1'+
r'\2018_04_30-11_38\Analysis\Tsne\Video\CroppedVideo_100ms_Top100PCs',
'47.2': r'Y:\swc\kampff\George\DataAndResults\Experiments\Awake\NeuroSeeker\AK_47.2'+
r'\2019_06_25-12_50\Analysis\Tsnes\Video\CroppedVideo_100ms_Top100PCs',
'47.1': r'Y:\swc\kampff\George\DataAndResults\Experiments\Awake\NeuroSeeker\AK_47.1'+
r'\2019_07_04-11_51\Analysis\Tsnes\Video\CroppedVideo_100ms_Top100PCs'}
tsne_results_nn = {}
tsne_results_sp = {}
video_labels = {}
for r in rats:
rat = rats[r]
tsne_results_nn[rat] = tsne_io.load_tsne_result(tsne_folders_nn[rat])
tsne_results_sp[rat] = tsne_io.load_tsne_result(tsne_folders_sp[rat])
video_labels[rat] = np.load(join(video_tsne_folders[rat], 'dbscan_labels.npy'))
mean_distances_of_labels_nn = {}
mean_distances_of_labels_sp = {}
for r in rats:
rat = rats[r]
unique_labels = np.unique(video_labels[rat])
unique_labels = np.delete(unique_labels, np.argwhere(unique_labels==-1))
nn_tsne = tsne_results_nn[rat]
random_selection_of_tsne_points_indices = np.random.choice(np.arange(len(nn_tsne)), 20000)
random_selection_of_tsne_points = nn_tsne[random_selection_of_tsne_points_indices, :]
template_features_sparse_clean = np.load(join(tsne_folder, 'data_to_tsne_(272886, 140).npy'))
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release'
theta = 0.2
eta = 200.0
num_dims = 2
perplexity = 100
iterations = 4000
random_seed = 1
verbose = 3
tsne = TSNE.t_sne(samples=template_features_sparse_clean, files_dir=tsne_folder, exe_dir=exe_dir, num_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, iterations=iterations, random_seed=random_seed,
verbose=verbose)
tsne = io.load_tsne_result(tsne_folder)
# and plotting results ------------------------------
spike_clusters_clean = spike_clusters[spikes_used]
labels_dict = {}
clean_templates = np.argwhere(template_marking > 0)
total_spikes = 0
for i in np.arange(len(clean_templates)):
spikes = np.squeeze(np.argwhere(np.in1d(spike_clusters_clean, clean_templates[i])))
labels_dict[i] = spikes
pf.plot_tsne(tsne.T, labels_dict=labels_dict, legent_on=False)
# ---------------------------------------------------
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release'
theta = 0.4
eta = 200.0
exageration = 12
num_dims = 2
perplexity = 100
iterations = 4000
random_seed = 1
verbose = 3
tsne = TSNE.t_sne(samples=template_features_sparse_clean, files_dir=files_dir, exe_dir=exe_dir, num_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, exageration=exageration, iterations=iterations,
random_seed=random_seed, verbose=verbose)
# OR LOAD
tsne = io.load_tsne_result(files_dir=files_dir)
# -------------------------------------------------------------------
# PLOTTING
spike_templates = np.load(join(base_folder, 'spike_templates.npy'))
template_markings = np.load(join(base_folder, 'template_marking.npy'))
clean_templates_index = np.argwhere(template_markings > 0)
clean_spikes_index = np.squeeze(np.argwhere(np.in1d(spike_templates, clean_templates_index)))
np.save(join(files_dir, 'indices_of_spikes_used.npy'), clean_spikes_index)
spikes_used = np.load(join(files_dir, 'indices_of_spikes_used.npy'))
spike_templates_clean = spike_templates[spikes_used]
def create_cluster_info_from_kilosort_spike_templates(cluster_info_filename, spike_templates):
mutual_information_folder = join(analysis_folder, 'Results',
'MutualInformation')
template_info = pd.read_pickle(join(kilosort_folder, 'template_info.df'))
time_to_bin = 0.1
frames_to_bin = time_to_bin * 120
# Define the folder for the t-snes
tsne_video_folder = join(tsne_video_folder_base,
'CroppedVideo_100ms_Top100PCs')
tsne_spikes_folder = join(tsne_spikes_folder_base,
'All_spikes_100msbin_count_top40PCs_10Kiter')
# Load t-sne specific stuff
tsne_spikes = tsne_io.load_tsne_result(tsne_spikes_folder)
tsne_pcs_spike_count_image_file = join(
analysis_folder, 'Images',
'Tsne_of_40_top_PCs_of_0p1_count_spike_vectors.png')
tsne_pcs_spike_count_image = plt.imread(tsne_pcs_spike_count_image_file)
tsne_pcs_spike_count_image_extent = [-16, 21, -17, 14]
tsne_video = tsne_io.load_tsne_result(tsne_video_folder)
dbscan_result_file = join(tsne_video_folder, 'dbscan_result.pcl')
dbscan_result = pickle.load(open(dbscan_result_file, 'rb'))
tsne_pcs_video_image_file = join(tsne_video_folder,
'Tsne_of_100_top_PCs_of_cropped_video.png')
tsne_pcs_video_image = plt.imread(tsne_pcs_video_image_file)
number_of_top_pcs = 40
num_dims = 2
perplexity = 30
theta = 0.3
eta = 200
exageration = 12
iterations = 4000
random_seed = 1
verbose = 2
tsne_result_full = tsne.t_sne(pcs_ar_full[:, :number_of_top_pcs], tsne_folder_full, barnes_hut_exe_dir,
num_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, exageration=exageration,
iterations=iterations, random_seed=random_seed, verbose=verbose)
tsne_result_full = tsne_io.load_tsne_result(tsne_folder_full)
plt.figure(1)
plt.scatter(tsne_result_full[:, 0], tsne_result_full[:, 1], s=3, c='k')
plt.figure(2)
colors = ['k', 'r', 'b', 'y', 'g', 'm']
for ind, i in zip(indices_full, np.arange(len(indices_full))):
plt.scatter(tsne_result_full[ind[0]:ind[1], 0], tsne_result_full[ind[0]:ind[1], 1], s=10, c=colors[i])
plt.legend(cell_folders)
# T-sne the decimated
tsne_folder_deci = join(base_save_folder, 'common_results', 'tsne', 'PCs_of_decimated_waveforms')
spike_rates_per_video_frame_filename = join(
kilosort_folder, 'firing_rate_with_video_frame_window.npy')
spike_rates_per_video_frame = np.load(spike_rates_per_video_frame_filename)
dlc_folder = join(analysis_folder, 'Deeplabcut')
dlc_project_folder = join(dlc_folder, 'projects', 'V1--2019-06-30')
video_file = join(dlc_folder, 'BonsaiCroping', 'Full_video.avi')
body_positions = np.load(
join(dlc_project_folder, 'post_processing', 'body_positions.npy'))
speeds = np.load(join(dlc_project_folder, 'post_processing', 'speeds.npy'))
patterned_vs_non_patterned_folder = join(analysis_folder, 'Behaviour',
'PatternedVsNonPatterned')
results_folder = join(analysis_folder, 'Results')
poke_folder = join(results_folder, 'EventsCorrelations', 'Poke')
events_definitions_folder = join(results_folder, 'EventsDefinitions')
template_info = pd.read_pickle(join(kilosort_folder, 'template_info.df'))
tsne_folder = join(tsne_folder_base,
'All_spikes_100msbin_count_top40PCs_10Kiter')
tsne_result = tsne_io.load_tsne_result(tsne_folder).transpose()
# </editor-fold>
# -------------------------------------------------
db, n_clusters_, labels, core_samples_mask, score = tsne_funcs.fit_dbscan(
tsne_result, eps=0.032, min_samples=150, normalize=False, show=True)
