def flatten_tcs(A,dof='EstEff'):
""" Concat timecourses, demeaning, estimating dof """
tcs_dm=pl.demean(A.tcs,2)
shp=tcs_dm.shape
out=FC((np.reshape(tcs_dm.swapaxes(0,1),[shp[1],-1])),ROI_info=A.ROI_info)
# subtract dofs from demeaning
out.dof=out.dof-shp[0]
tcs = out.tcs
if dof is None:
self.dof=tcs.shape[-1]-1
elif dof == 'EstEff':
AR=np.zeros((tcs.shape[0],tcs.shape[1],15))
ps=np.zeros((tcs.shape[0],tcs.shape[1],15))
for subj in np.arange(tcs.shape[0]):
for ROI in np.arange(tcs.shape[1]):
AR[subj,ROI,:]=spectrum.aryule(pl.demean(tcs[subj,ROI,:]),15)[0]
ps[subj,ROI,:]=spectrum.correlation.CORRELATION(pl.demean(tcs[subj,ROI,:]),maxlags=14,norm='coeff')
ps = np.mean(np.mean(ps,0),0)
AR = np.mean(np.mean(AR,0),0)
dof_nom=tcs.shape[-1]-1
dof = int(dof_nom / (1-np.dot(ps[:15].T,AR))/(1 - np.dot(np.ones(len(AR)).T,AR)))
out.dof=dof
return(out)
def seed_loader(filenames,seed_mask_file,mask_file,subj_inds=None,dof=None):
""" load data for seed analysis """
files=[]
seed=nb.load(seed_mask_file)
seed_mask = seed.get_data()
seed_points = where(seed_mask)
mask=nb.load(mask_file)
mask_data = mask.get_data()
mask_points = where(mask_data)
covmat=coo_matrix((len(mask_points[0])+1,len(mask_points[0])+1)).tocsr()
if type(filenames)==str:
filenames=[filenames]
dof_cnt=0
for file in filenames:
print(file)
newfile = nb.load(file)
files.append(newfile)
# load
data=newfile.get_data()
seed_data = data[seed_points[0],seed_points[1],seed_points[2],:]
seed_mean = pl.demean(np.mean(seed_data,0))
data_mask = data[mask_points[0],mask_points[1],mask_points[2],:]
vars = var(data_mask,1)
covs = sum(seed_mean*pl.demean(data_mask,1),1)/len(seed_mean)
# corrs = sum(seed_mean*pl.demean(data_mask,1),1)/(np.std(seed_mean)*np.std(data_mask,1)*len(seed_mean))
# data: corner, top row, first colomn, diag
rows=np.r_[0,np.zeros(vars.shape),np.arange(len(vars))+1,np.arange(len(vars))+1]
cols=np.r_[0,np.arange(len(vars))+1,np.zeros(vars.shape),np.arange(len(vars))+1]
covmat_data=np.r_[var(seed_mean),covs,covs,vars]
covmat = covmat + coo_matrix((covmat_data,(rows,cols)),shape=(len(vars)+1,len(vars)+1)).tocsr()
dof_cnt = dof_cnt+seed_data.shape[-1]-1
# FC_cov = FC(covmat)
newfile.uncache()
if not dof:
dof=dof_cnt
covmat=covmat/len(filenames)
out = FC(covmat,cov_flag=True,dof=dof,mask=mask)
return out
def get_covs(A,B=None):
""" get covariances from FC. """
if B is None:
covs=np.zeros((A.shape[0],A.shape[1],A.shape[1]))
for a in np.arange(A.shape[0]):
covs[a,:,:]=np.cov(A[a,:,:])
else:
covs = sum(pl.demean(A,2)*pl.demean(B,2),2)/(A.shape[2])
return covs
def __init__(self,tcs,cov_flag=False,dof=None,ROI_info={},mask=None):
""" Initialise the FC object. """
# reading covariance matrix
if cov_flag==True:
self.tcs=None
covs=tcs
if dof is None:
raise ValueError("Need to specify dof if providing cov.")
else:
self.dof=dof
else:
# reading timecourses
if tcs.ndim==2:
tcs=(np.atleast_3d(tcs).transpose(2,0,1))
self.tcs = tcs
if dof is None:
self.dof=tcs.shape[-1]-1
elif dof == 'EstEff':
AR=np.zeros((tcs.shape[0],tcs.shape[1],15))
ps=np.zeros((tcs.shape[0],tcs.shape[1],15))
for subj in np.arange(tcs.shape[0]):
for ROI in np.arange(tcs.shape[1]):
AR[subj,ROI,:]=spectrum.aryule(pl.demean(tcs[subj,ROI,:]),15)[0]
ps[subj,ROI,:]=spectrum.correlation.CORRELATION(pl.demean(tcs[subj,ROI,:]),maxlags=14,norm='coeff')
ps = np.mean(np.mean(ps,0),0)
AR2 = np.mean(np.mean(AR,0),0)
dof_nom=tcs.shape[-1]-1
self.dof = int(dof_nom / (1-np.dot(ps[:15].T,AR2))/(1 - np.dot(np.ones(len(AR2)).T,AR2)))
else:
self.dof=dof
covs = get_covs(tcs)
if ROI_info=={}:
shp=covs.shape[-1]
ROI_info['ROI_names']=np.arange(shp).astype(str)
ROI_info['ROI_RSNs']=np.ones(shp,).astype(int)
if mask:
self.mask=mask
self.ROI_info = ROI_info
self.covs = covs
def get_corrs(A,B=None,pcorrs=False):
""" get correlations from FC. """
if B is None:
covs=get_covs(A)
stds=A.get_stds()
stds_m =np.tile(stds,(1,covs.shape[1])).reshape(stds.shape[0],stds.shape[1],stds.shape[1])
stds_m_t = transpose(stds_m,(0,2,1))
corrs=covs/(stds_m*stds_m_t)
else:
corrs = sum(pl.demean(A,2)*pl.demean(B,2),2)/(A.shape[2]*np.std(A,2)*np.std(B,2))
return corrs
def t_sne(samples, files_dir=None, exe_dir=None, num_dims=2, perplexity=100, theta=0.4, eta=200, exageration=12.0,
iterations=1000, random_seed=-1, verbose=2):
"""
Runs the t-SNE algorithm on the samples matrix
:param samples: the samples x features matrix to run t-SNE on
:type samples: float32[:,:]
:param files_dir: where the data.dat file is and where the t-SNE will save its result
:type files_dir: string
:param exe_dir: where the Barnes_Hut.exe is (if None then the algorithm will try and find it itself)
:type exe_dir: string
:param num_dims: how many dimensions should the t-sne embedding have (2 or 3)
:type num_dims: int
:param perplexity: the number of closest spikes considered are 3 * perplexity
:type perplexity: int
:param theta: the angle on the t-sne embedding inside of which all spikes are considered a single point (0 to 1)
:type theta: float
:param eta: the learning rate of the algorithm
:type eta: float
:param exageration: the initial blow out (push out) of the points on the embedding
:type exageration: float
:param iterations: the number of iterations of the t-sne algorithm
:type iterations: int
:param random_seed: the random seed to create the initial random positions of the points on the embedding
:type random_seed: float
:param verbose: levels of verbosity. 0 is no output. If 3 then the algorithm will also save all intermediate t-sne
embeddings. Useful for making videos of the t-sne process
:type verbose: int
:return: the t-NSE result
:rtype: float32[:,num_of_dims]
"""
data = pylab.demean(samples, axis=0)
data /= data.max()
closest_indices_in_hd, closest_distances_in_hd = \
gpu.calculate_knn_distances(samples_matrix=data, perplexity=perplexity, verbose=True)
io.save_data_for_barneshut(files_dir, closest_distances_in_hd, closest_indices_in_hd, num_of_dims=num_dims,
perplexity=perplexity, theta=theta, eta=eta, exageration=exageration,
iterations=iterations, random_seed=random_seed, verbose=verbose)
del samples
exe_file = get_barnes_hut_executable(exe_dir)
run_executable(exe_file, files_dir, verbose)
tsne = io.load_tsne_result(files_dir)
return tsne
def tsdata_to_var(X, p):
import numpy as np
from matplotlib import pylab
X = X[:,:,np.newaxis]
[n, m, N] = X.shape
N=1
# assert(p < m,'too many lags');
p1 = p+1
A = 0 #nan
SIG = 0 #nan
E = 0 #nan
X = pylab.demean(X, axis=1)
# no constant term
M = np.dot(N, m-p)
n_p = np.dot(n, p)
# stack lags
X0 = np.reshape(X[:,p1-1:m], (n, M))
# concatenate trials for unlagged observations
XL = np.zeros((n, p, M))
for k in range(1, (p)+1): #if lag = 1, only 1 iteration
XL[:,k-1,:] = np.reshape(X[:,p1-k-1:m-k,:], (n, M))
# concatenate trials for k-lagged observations
XL = np.reshape(XL, (n_p, M))
# stack lags
A = np.linalg.lstsq(XL.T,X0.T)[0].T
# so A(:,:,k) is the k-lag coefficients matrix
# OLS using QR decomposition
E = X0-np.dot(A, XL)
# residuals
SIG = np.dot(E, E.T)/(M-1)
# residuals covariance matrix
return [A, SIG, E]
print(s)
s+=1
for i in np.arange(len(indices)):
template_features_sparse_clean[np.argwhere(spikes_clean_index == spike),
np.argwhere(clean_templates == indices[i])] = template_features[
spike, i]
np.save(join(base_folder, 'data_to_tsne_' + str(template_features_sparse_clean.shape) + '.npy'),
template_features_sparse_clean)
# --------
template_features_sparse_clean = np.load(join(base_folder, r'data_to_tsne_(20000, 172).npy'))
# --------
data = pylab.demean(template_features_sparse_clean, axis=0)
data /= data.max()
perplexity = 100
closest_indices_in_hd, closest_distances_in_hd = \
gpu.calculate_knn_distances_all_to_all(template_features_sparse_clean=data, perplexity=perplexity, mem_usage=0.9,
verbose=True)
# PASS THE SORTED DISTANCES TO BARNES HUT C++ TO GENERATE T-SNE RESULTS
theta = 0.4
eta = 200.0
num_dims = 2
iterations = 3000
verbose = 3
exe_folder = r'E:\Software\Develop\Source\Repos\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release' # Desktop
#exe_folder = r'E:\Projects\Analysis\Brain\spikesorting_tsne_bhpart\Barnes_Hut\x64\Release' # Laptop
io.save_data_for_barneshut(exe_folder, closest_distances_in_hd, closest_indices_in_hd, eta=eta, iterations=iterations,
template_features_sparce_clean_probesorted = template_features_sparse_clean[
spike_indices_sorted_by_probe_distance, :]
return indices_of_first_arrays, indices_of_second_arrays, \
spike_indices_sorted_by_probe_distance, template_features_sparce_clean_probesorted
# GET SOME TEMPLATE FEATURES DATA
base_folder = r'E:\Data\Brain\Neuroseeker_2016_12_17_Anesthesia_Auditory_DoubleProbes\AngledProbe\Kilosort results'
binary_data_filename = r'AngledProbe_BinaryAmplifier_12Regions_Penetration1_2016-12-17T19_02_12.bin'
number_of_spikes = 100000
indices_of_first_arrays, indices_of_second_arrays, \
spike_indices_sorted_by_probe_distance, template_features_sparce_clean_probesorted = \
get_some_data(base_folder=base_folder, binary_data_filename=binary_data_filename, number_of_spikes=number_of_spikes)
data = pylab.demean(template_features_sparce_clean_probesorted, axis=0)
data /= data.max()
# CALCULATE DISTANCES ON THE GPU
indices_of_first_and_second_matrices = (indices_of_first_arrays,
indices_of_second_arrays)
perplexity = 100
theta = 0.4
eta = 200.0
num_dims = 2
iterations = 1000
verbose = 2
num_samples = data.shape[0]
closest_indices_in_hd, closest_distances_in_hd = \