templates_in_cortex['number of spikes'].sum()
spikes_in_cortex = np.array([])
for t in templates_in_cortex['spikes in template']:
spikes_in_cortex = np.concatenate((spikes_in_cortex, t))
# -------------------------------------------------------------------------
# Tsne all spikes in cortex together
# Use the distance to templates as features
tsne_cortex_folder = join(denoised_folder, 'Tsne', 'Cortex')
np.save(join(tsne_cortex_folder, 'indices_of_spikes_used.npy'),
spikes_in_cortex)
template_features = preproc_kilo.calculate_template_features_matrix_for_tsne(
kilosort_folder_denoised,
tsne_cortex_folder,
spikes_used_with_original_indexing=spikes_in_cortex)
num_dims = 2
perplexity = 100
theta = 0.3
iterations = 4000
random_seed = 1
verbose = 2
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\win\x64\Release'
tsne_results = tsne.t_sne(template_features,
files_dir=tsne_cortex_folder,
exe_dir=exe_dir,
num_dims=num_dims,
perplexity=perplexity,
theta=theta,
# ----------------------------------------------------------------------------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------
# <editor-fold desc = STEP 3: T-SNE THE TEMPLATES
# a) Pick the good spikes
spikes_of_good_templates = np.array([])
for t in template_info['spikes in template']:
spikes_of_good_templates = np.concatenate((spikes_of_good_templates, t))
# b) Make the distance of each spike to its template's closest templates as the feature matrix of T-sne
np.save(join(tsne_folder, 'indices_of_spikes_used.npy'), spikes_of_good_templates) # Very important to save this file!
template_features = preproc_kilo.calculate_template_features_matrix_for_tsne(kilosort_folder,
tsne_folder,
spikes_used_with_original_indexing=spikes_of_good_templates)
# c) Run the T-sne
num_dims = 2
perplexity = 100
theta = 0.3
iterations = 4000
random_seed = 1
verbose = 2
# This is a full T-sne (calculates distances in GPU and the T-sne embedding in CPU)
tsne_results = tsne.t_sne(template_features, files_dir=tsne_folder, exe_dir=tsne_exe_dir, num_dims=num_dims,
perplexity=perplexity, theta=theta, iterations=iterations, random_seed=random_seed,
verbose=verbose)
files_dir = join(base_folder, 'Tsne_Results', '2017_08_20_dis700K_579templates')
total_spikes_required = 700000
full_inclusion_threshold = 15000 # number of spikes or fewer for a template to have to be fully included
spikes_used, small_clean_templates_with_spike_indices, large_clean_templates_with_spike_indices\
= preproc.find_spike_indices_for_representative_tsne(base_folder=base_folder,
save_to_folder=files_dir,
threshold=full_inclusion_threshold,
total_spikes_required=total_spikes_required)
spikes_used = None # Use this to load all spikes for tsne
template_features_sparse_clean = \
preproc.calculate_template_features_matrix_for_tsne(base_folder, save_to_folder=files_dir,
spikes_used_with_original_indexing=spikes_used)
# OR Load it
template_features_sparse_clean = np.load(join(files_dir, 'data_to_tsne_(1091229, 579).npy'))
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release'
theta = 0.4
eta = 200.0
exageration = 20
num_dims = 2
perplexity = 100
iterations = 4000
random_seed = 1
verbose = 3
tsne = TSNE.t_sne(samples=template_features_sparse_clean, files_dir=files_dir, exe_dir=exe_dir, num_dims=num_dims,
number_of_raw_spikes = len(templates_of_spikes)
template_markings = preproc_kilo.get_template_marking(kilosort_folder)
clean_templates = np.argwhere(template_markings > 0)
number_of_clean_templates = len(clean_templates)
clean_spikes = np.argwhere(np.in1d(templates_of_spikes, clean_templates) > 0)
number_of_clean_spikes = len(clean_spikes)
# Get a representative sample of spikes for t-sne
representative_indices, small_templates, large_templates = \
preproc_kilo.find_spike_indices_for_representative_tsne(kilosort_folder, tsne_folder, 5000, 700000)
template_features_matrix = preproc_kilo.calculate_template_features_matrix_for_tsne(kilosort_folder, tsne_folder,
spikes_used_with_original_indexing=
representative_indices)
template_features_matrix = np.load(join(tsne_folder, 'data_to_tsne_(699973, 762).npy'))
# First run of t-sne
num_dims = 2
perplexity = 100
theta = 0.3
iterations = 4000
random_seed = 1
verbose = 3
tsne_results = tsne.t_sne(template_features_matrix, files_dir=tsne_folder, num_dims=num_dims, perplexity=perplexity,
theta=theta, iterations=iterations, random_seed=random_seed, verbose=verbose)
# ---------------------------------------------------
# Generate positions on probe
weighted_template_positions = pos.generate_probe_positions_of_templates(kilosort_folder, threshold=0.1)
np.save(join(kilosort_folder, 'weighted_template_positions.npy'), weighted_template_positions)
# ---------------------------
# REDOING TSNE ON CLEAN KILOSORT RESULTS ------------
spike_clusters = np.load(join(kilosort_folder, 'spike_clusters.npy'))
template_marking = np.load(join(kilosort_folder, 'template_marking.npy'))
spikes_used = np.array([spike_index for spike_index in np.arange(len(spike_clusters)) if template_marking[spike_clusters[spike_index]] > 0])
np.save(join(tsne_folder, 'indices_of_spikes_used.npy'), spikes_used)
template_features_sparse_clean = \
preproc.calculate_template_features_matrix_for_tsne(kilosort_folder, save_to_folder=tsne_folder,
spikes_used_with_original_indexing=spikes_used)
template_features_sparse_clean = np.load(join(tsne_folder, 'data_to_tsne_(272886, 140).npy'))
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release'
theta = 0.2
eta = 200.0
num_dims = 2
perplexity = 100
iterations = 4000
random_seed = 1
verbose = 3
tsne = TSNE.t_sne(samples=template_features_sparse_clean, files_dir=tsne_folder, exe_dir=exe_dir, num_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, iterations=iterations, random_seed=random_seed,
verbose=verbose)
import pandas as pd
from os.path import join
import numpy as np
import matplotlib.pyplot as plt
from spikesorting_tsne import io_with_cpp as io
from BrainDataAnalysis.Graphics import ploting_functions as pf
base_folder = r'Z:\n\Neuroseeker Probe Recordings\Neuroseeker_2017_08_08\Analysis\kilosort\18_26_30_afterREFeachGroup'
files_dir = join(base_folder, 'Tsne_Results')
# Get features of all clean spikes
template_features_sparse_clean = \
preproc.calculate_template_features_matrix_for_tsne(base_folder, save_to_folder=files_dir)
# OR Load it
template_features_sparse_clean = np.load(join(files_dir, 'data_to_tsne_(513722, 588).npy'))
exe_dir = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release'
theta = 0.4
eta = 200.0
exageration = 12
num_dims = 2
perplexity = 100
iterations = 4000
random_seed = 1
verbose = 3
tsne = TSNE.t_sne(samples=template_features_sparse_clean, files_dir=files_dir, exe_dir=exe_dir, num_dims=num_dims,