def test_ppf_vf(self):
te = dbfn.TaskEntry(2425, dbname='testing')
dec_record = te.decoder_record
dec = dec_record.load()
training_block = dbfn.TaskEntry(dec_record.entry_id, dbname='testing')
files = training_block.datafiles
from riglib.bmi import extractor
extractor_cls = extractor.BinnedSpikeCountsExtractor
extractor_kwargs = dict()
extractor_kwargs['units'] = dec.units
extractor_kwargs['n_subbins'] = dec.n_subbins
dec_new = train.train_PPFDecoder(files, extractor_cls, extractor_kwargs, train.get_plant_pos_vel, dec.ssm, dec.units, update_rate=dec.binlen, kin_source='task', pos_key='cursor', vel_key=None, tslice=dec.tslice)
for attr in dec.filt.model_attrs:
old_attr = getattr(dec.filt, attr)
new_attr = getattr(dec_new.filt, attr)
self.assertTrue(np.all((old_attr - new_attr) < 1e-10))
self.assertTrue(np.array_equal(dec_new.units, dec.units))
self.assertTrue(np.array_equal(dec_new.bounding_box[0], dec.bounding_box[0]))
self.assertTrue(np.array_equal(dec_new.bounding_box[1], dec.bounding_box[1]))
self.assertTrue(dec_new.states == dec.states)
self.assertTrue(dec_new.states_to_bound == dec.states_to_bound)
self.assertTrue(np.array_equal(dec_new.drives_neurons, dec.drives_neurons))
def save_new_dec(task_entry_id, dec_obj, suffix):
'''
Summary: Method to save decoder to DB -- saves to TE that original decoder came from
Input param: task_entry_id: original task to save decoder to
Input param: dec_obj: KF decoder new
Input param: suffix:
Output param:
'''
te = dbfn.TaskEntry(task_entry_id)
try:
te_id = te.te_id
except:
dec_nm = te.name
te_ix = re.search('te[0-9]',dec_nm)
ix = te_ix.start() + 2
sub_dec_nm = dec_nm[ix:]
te_ix_end = sub_dec_nm.find('_')
if te_ix_end == -1:
te_ix_end = len(sub_dec_nm)
te_id = int(sub_dec_nm[:te_ix_end])
old_dec_obj = te.decoder_record
if old_dec_obj is None:
old_dec_obj = faux_decoder_obj(task_entry_id)
trainbmi.save_new_decoder_from_existing(dec_obj, old_dec_obj, suffix=suffix)
def _get_te(te, **kwargs):
# dbname = kwargs.pop('dbname', 'default')
te = dbfn.TaskEntry(te, **kwargs)
try:
return tasks[te.record.task.name](te.record.id, **kwargs)
except:
return te
def get_seed_block(te, max_depth=np.inf):
'''
Determine how much time was spent on CLDA for a given decoder on a particular day
'''
decoder_record = te.decoder_record
from db.tracker import models
depth = 0
total_time = 0
while depth < max_depth:
rec = models.TaskEntry.objects.using(
te.record._state.db).get(id=decoder_record.entry_id)
t = datetime.datetime.strptime(
rec.offline_report()['Runtime'],
r'%H:%M:%S') - datetime.datetime.strptime('', '')
total_time += t.seconds
try:
new_decoder_record = dbfn.TaskEntry(
rec, dbname=te.record._state.db).decoder_record
if new_decoder_record == decoder_record:
print 'circle'
break
else:
decoder_record = new_decoder_record
if decoder_record.entry_id == None:
break
except:
break
depth += 1
return rec
def get(id, task_specific=True, **kwargs): if task_specific: import performance return performance._get_te(id, **kwargs) else: from db import dbfunctions return dbfunctions.TaskEntry(id)
def get_decoder_corr(task_entry_id, decoder_entry_id, get_dec_used=True):
'''
Summary: get KF decoder either from entry that has trained the decoder (if this, need decoder_entry_id if > 1 decoder),
or decoder that was used during task_entry_id
Input param: task_entry_id: dbname task entry ID
Input param: decoder_entry_id: decoder entry id: (models.Decoder.objects.get(entry=entry))
Output param: KF Decoder
'''
ld = True
if get_dec_used is False:
decoder_entries = dbfn.TaskEntry(
task_entry_id).get_decoders_trained_in_block()
if len(decoder_entries) > 0:
print 'Loading decoder TRAINED from task %d' % task_entry_id
if type(decoder_entries) is models.Decoder:
decoder = decoder_entries
ld = False
else: # list of decoders. Search for the right one.
try:
dec_ids = [de.pk for de in decoder_entries]
_ix = np.nonzero(dec_ids == decoder_entry_id)[0]
decoder = decoder_entries[_ix]
ld = False
except:
if decoder_entry_id is None:
print 'Too many decoder entries trained from this TE, specify decoder_entry_id'
else:
print 'Too many decoder entries trained from this TE, no match to decoder_entry_id %d' % decoder_entry_id
if ld is False:
kfdec = decoder.load()
else:
try:
kfdec = dbfn.TaskEntry(task_entry_id).decoder
print 'Loading decoder USED in task %s' % dbfn.TaskEntry(
task_entry_id).task
except:
raise Exception('Cannot load decoder from TE%d' % task_entry_id)
return kfdec
def get_decoder_bias(decoder_name):
if isinstance(decoder_name, int):
decoder_name = dbf.TaskEntry(decoder_name,
db_name=dbf.db_name).decoder_record.name
decoder = models.Decoder.objects.using(dbf.db_name).get(name=decoder_name)
decoder = decoder.load(db_name=dbf.db_name)
if 'sskf' in decoder_name:
#if isinstance(decoder, sskfdecoder.SSKFDecoder): #'sskf' in decoder_name:
unbiased_name = decoder_name[:-6]
unbiased_decoder = models.Decoder.objects.get(
name=unbiased_name).load()
bias_vec = decoder.filt.F[3:6, -1] - unbiased_decoder.filt.F[3:6, -1]
else:
bias_vec = np.asarray(decoder.filt.A)[3:6, -1]
gain = np.linalg.norm(bias_vec)
angle = np.arctan2(bias_vec[-1], bias_vec[0])
if angle < 0: angle += 2 * np.pi
return dict(gain=gain, angle=angle * 180 / np.pi)
def offline_report(self):
Exp = self.task.get(self.feats.all())
if len(self.report) == 0:
return dict()
else:
report = json.loads(self.report)
rpt = Exp.offline_report(report)
## If this is a BMI block, add the decoder name to the report (doesn't show up properly in drop-down menu for old blocks)
try:
from db import dbfunctions
te = dbfunctions.TaskEntry(self.id, dbname=self._state.db)
rpt['Decoder name'] = te.decoder_record.name + ' (trained in block %d)' % te.decoder_record.entry_id
except AttributeError:
pass
except:
import traceback
traceback.print_exc()
return rpt
#!/usr/bin/python
'''
A set of tests to ensure that all the dbfunctions are still functional
'''
import os
import django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'db.settings')
django.setup()
from db.tracker import models
import db.dbfunctions as dbfn
id = 77
te = dbfn.TaskEntry(id)
print(dbfn.get_date(id))
print(dbfn.get_notes(id))
print(dbfn.get_subject(id))
print(dbfn.get_length(id))
print(dbfn.get_success_rate(id))
#print(dbfn.get_plx_file(id))
#print(dbfn.get_decoder_name(id))
#print(dbfn.get_decoder_name_full(id))
#print(dbfn.get_decoder(id))
#print(dbfn.get_params(id))
#print(dbfn.get_param(id, 'decoder'))
#id = 1956
#print(dbfn.get_bmiparams_file(id))
import os
import django
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'db.settings')
django.setup()
from db.tracker import models
fe = models.Feature.objects.all()
print(fe)
ID_NUMBER = 77
te = models.TaskEntry.objects.get(id=ID_NUMBER)
print(te.subject)
# try a different method
# need to take out the dot in the .tracker import
# this method gets depreciated
import db.dbfunctions as dbfn
te_1 = dbfn.TaskEntry(ID_NUMBER)
print(te_1.date)
'''
'''
#Open new file:
f = open(os.path.expandvars('$HOME/files_to_backup'), 'w')
#Append to previous file:
f_list = open(os.path.expandvars('$HOME/files_to_backup_te_list'), 'a')
for k, te in enumerate(backed_up_tes):
#Check if already backed up:
if te.id in already_te_added:
pass
else:
#Array for datafile names:
datafiles = []
try:
te = dbfn.TaskEntry(te)
te_datafiles = te.datafiles
for sysname in systems_to_back_up:
if sysname in te_datafiles:
datafiles.append(te_datafiles[sysname])
#Close HDF file to free memory:
try:
te.hdf.close()
except:
pass
if k % 100 == 0:
print(k)
if not te.decoder_record is None:
def training_tau(self):
try:
return dbfn.TaskEntry(self.decoder_record.entry).params['tau']
except:
return np.nan
def __init__(self,
n_iter=None,
entry_id=None,
hdf_filename=None,
decoder_filename=None,
params=dict(),
*args,
**kwargs):
if entry_id is None and (hdf_filename is None
or decoder_filename is None):
raise ValueError(
"Not enough data to reconstruct a BMI! Specify a database entry OR an HDF file + decoder file"
)
if entry_id is not None:
from db import dbfunctions as dbfn
te = dbfn.TaskEntry(entry_id)
self.hdf_ref = te.hdf
self.decoder = te.decoder
self.params = te.params
if self.hdf_ref is None:
raise ValueError("Database is unable to locate HDF file!")
if self.decoder is None:
raise ValueError("Database is unable to locate HDF file!")
elif hdf_filename is not None:
self.hdf_ref = tables.open_file(hdf_filename)
self.decoder = pickle.load(
open(decoder_filename, 'rb'),
encoding='latin1') # extra args to get py3 to read py2 pickles
self.params = params
self.n_iter = min(n_iter, len(self.hdf_ref.root.task))
try:
self.starting_pos = self.hdf_ref.root.task[0]['decoder_state'][0:3,
0]
except:
# The statement above appears to not always work...
self.starting_pos = self.hdf_ref.root.task[0][
'cursor'] # #(0, 0, 0)
# if 'plant_type' in te.params:
# self.plant_type = te.params['plant_type']
# elif 'arm_class' in te.params:
# plant_type = te.params['arm_class']
# if plant_type == 'CursorPlant':
# self.plant_type = 'cursor_14x14'
# else:
# self.plant_type = plant_type
# else:
# self.plant_type = 'cursor_14x14'
# TODO overly specific
self.plant = plants.CursorPlant(endpt_bounds=(-14, 14, 0., 0., -14,
14))
## Set the target radius because the old assist method changes the assist speed
# when the cursor is inside the target
self.target_radius = params['target_radius']
self.cursor_radius = params['cursor_radius']
self.assist_level = params['assist_level']
self.idx = 0
gen = sim_target_seq_generator_multi(8, 1000)
super(BMIReconstruction, self).__init__(gen, *args, **kwargs)
self.hdf = SimHDF()
self.learn_flag = True
task_msgs = self.hdf_ref.root.task_msgs[:]
self.update_bmi_msgs = task_msgs[task_msgs['msg'] == 'update_bmi']
task_msgs = list(
filter(lambda x: x['msg'] not in ['update_bmi'], task_msgs))
# print task_msgs
self.task_state = np.array([None] * n_iter)
for msg, next_msg in zip(task_msgs[:-1], task_msgs[1:]):
self.task_state[msg['time']:next_msg['time']] = msg['msg']
self.update_bmi_inds = np.zeros(len(self.hdf_ref.root.task))
self.update_bmi_inds[self.update_bmi_msgs['time']] = 1
self.recon_update_bmi_inds = np.zeros(len(self.hdf_ref.root.task))
self.target_hold_msgs = list(
filter(lambda x: x['msg'] in ['target', 'hold'],
self.hdf_ref.root.task_msgs[:]))
def generate_FA_matrices(self,
training_task_entry,
plot=False,
hdf=None,
dec=None,
bin_spk=None):
import utils.fa_decomp as pa
if bin_spk is None:
if training_task_entry is not None:
from db import dbfunctions as dbfn
te = dbfn.TaskEntry(training_task_entry)
hdf = te.hdf
dec = te.decoder
bin_spk, targ_pos, targ_ix, z, zz = self.extract_trials_all(
hdf, dec)
#Zscore is in time x neurons
zscore_X, mu = self.zscore_spks(bin_spk)
# #Find optimal number of factors:
LL, psv = pa.find_k_FA(zscore_X, iters=3, max_k=10, plot=False)
#Np.nanmean:
nan_ix = np.isnan(LL)
samp = np.sum(nan_ix == False, axis=0)
ll = np.nansum(LL, axis=0)
LL_new = np.divide(ll, samp)
num_factors = 1 + (np.argmax(LL_new))
print 'optimal LL factors: ', num_factors
FA = skdecomp.FactorAnalysis(n_components=num_factors)
#Samples x features:
FA.fit(zscore_X)
#FA matrices:
U = np.mat(FA.components_).T
i = np.diag_indices(U.shape[0])
Psi = np.mat(np.zeros((U.shape[0], U.shape[0])))
Psi[i] = FA.noise_variance_
A = U * U.T
B = np.linalg.inv(U * U.T + Psi)
mu_vect = np.array([mu[0, :]]).T #Size = N x 1
sharL = A * B
#Calculate shared / priv scaling:
bin_spk_tran = bin_spk.T
mu_mat = np.tile(np.array([mu[0, :]]).T, (1, bin_spk_tran.shape[1]))
demn = bin_spk_tran - mu_mat
shared_bin_spk = (sharL * demn)
priv_bin_spk = bin_spk_tran - mu_mat - shared_bin_spk
#Scaling:
eps = 1e-15
x_var = np.var(np.mat(bin_spk_tran), axis=1) + eps
pr_var = np.var(priv_bin_spk, axis=1) + eps
sh_var = np.var(shared_bin_spk, axis=1) + eps
priv_scalar = np.sqrt(np.divide(x_var, pr_var))
shared_scalar = np.sqrt(np.divide(x_var, sh_var))
if plot:
tmp = np.diag(U.T * U)
plt.plot(np.arange(1, num_factors + 1),
np.cumsum(tmp) / np.sum(tmp), '.-')
plt.plot([0, num_factors + 1], [.9, .9], '-')
#Get main shared space:
u, s, v = np.linalg.svd(A)
s_red = np.zeros_like(s)
s_hd = np.zeros_like(s)
ix = np.nonzero(np.cumsum(s**2) / float(np.sum(s**2)) > .90)[0]
if len(ix) > 0:
n_dim_main_shared = ix[0] + 1
else:
n_dim_main_shared = len(s)
if n_dim_main_shared < 2:
n_dim_main_shared = 2
print "main shared: n_dim: ", n_dim_main_shared, np.cumsum(s) / float(
np.sum(s))
s_red[:n_dim_main_shared] = s[:n_dim_main_shared]
s_hd[n_dim_main_shared:] = s[n_dim_main_shared:]
main_shared_A = u * np.diag(s_red) * v
hd_shared_A = u * np.diag(s_hd) * v
main_shared_B = np.linalg.inv(main_shared_A + hd_shared_A + Psi)
uut_psi_inv = main_shared_B.copy()
u_svd = u[:, :n_dim_main_shared]
main_sharL = main_shared_A * main_shared_B
main_shar = main_sharL * demn
main_shar_var = np.var(main_shar, axis=1) + eps
main_shar_scal = np.sqrt(np.divide(x_var, main_shar_var))
main_priv = demn - main_shar
main_priv_var = np.var(main_priv, axis=1) + eps
main_priv_scal = np.sqrt(np.divide(x_var, main_priv_var))
# #Get PCA decomposition:
#LL, ax = pa.FA_all_targ_ALLms(hdf, iters=2, max_k=20, PCA_instead=True)
#num_PCs = 1+(np.argmax(np.mean(LL, axis=0)))
# Main PCA space:
# Get cov matrix:
cov_pca = np.cov(zscore_X.T)
eig_val, eig_vec = np.linalg.eig(cov_pca)
tot_var = sum(eig_val)
cum_var_exp = np.cumsum(
[i / tot_var for i in sorted(eig_val, reverse=True)])
n_PCs = np.nonzero(cum_var_exp > 0.9)[0][0] + 1
proj_mat = eig_vec[:, :n_PCs]
proj_trans = np.mat(proj_mat) * np.mat(proj_mat.T)
#PC matrices:
return dict(fa_sharL=sharL,
fa_mu=mu_vect,
fa_shar_var_sc=shared_scalar,
fa_priv_var_sc=priv_scalar,
U=U,
Psi=Psi,
training_task_entry=training_task_entry,
FA_iterated_power=FA.iterated_power,
FA_score=FA.score(zscore_X),
FA_LL=np.array(FA.loglike_),
fa_main_shared=main_sharL,
fa_main_shared_sc=main_shar_scal,
fa_main_private_sc=main_priv_scal,
fa_main_shar_n_dim=n_dim_main_shared,
sing_vals=s,
own_pc_trans=proj_trans,
FA_model=FA,
uut_psi_inv=uut_psi_inv,
u_svd=u_svd)
def zscore_units(task_entry_id, calc_zscore_from_te, pos_key = 'cursor', decoder_entry_id=None,
training_method=train.train_KFDecoder, retrain_flag = False, **kwargs):
'''
Summary: Method to be able to 'convert' a trained decoder (that uses zscoring) to one that uses z-scored from another session
(e.g. you train a decoder from VFB, but you want to zscore unit according to a passive / still session earlier). You
would use the task_entry_id that was used to train the decoder OR entry that used the decoder. Then 'calc_zscore_from_te'
is the task entry ID used to compute the z-scored units. You can either retrain the decoder iwth the new z-scored units, or not
Input param: task_entry_id:
Input param: decoder_entry_id:
Input param: calc_zscore_from_te:
Output param:
'''
if 'decoder_path' in kwargs:
decoder = pickle.load(open(kwargs['decoder_path']))
else:
decoder = get_decoder_corr(task_entry_id, decoder_entry_id)
assert (hasattr(decoder, 'zscore') and decoder.zscore is True)," Cannot update mFR /sdFR of decoder that was not trained as zscored decoder. Retrain!"
# Init mFR / sdFR
if 'hdf_path' in kwargs:
hdf = tables.openFile(kwargs['hdf_path'])
else:
hdf = dbfn.TaskEntry(calc_zscore_from_te).hdf
# Get HDF update rate from hdf file.
try:
hdf_update_rate = np.round(np.mean(hdf.root.task[:]['loop_time'])*1000.)/1000.
except:
from config import config
if config.recording_system == 'blackrock':
hdf_update_rate = .05;
elif config.recording_system == 'plexon':
hdf_update_rate = 1/60.
spk_counts = hdf.root.task[:]['spike_counts'][:, :, 0]
# Make sure not repeated entries:
sum_spk_counts = np.sum(spk_counts, axis=1)
ix = np.nonzero(sum_spk_counts)[0][0]
sample = 1+ sum_spk_counts[ix:ix+6] - sum_spk_counts[ix]
assert np.sum(sample) != 6
decoder_update_rate = decoder.binlen
bin_spks, _ = bin_(None, spk_counts.T, hdf_update_rate, decoder_update_rate, only_neural=True)
mFR = np.squeeze(np.mean(bin_spks, axis=1))
sdFR = np.std(bin_spks, axis=1)
kwargs2 = dict(mFR=mFR, sdFR=sdFR)
#Retrain decoder w/ new zscoring:
if 'te_id' in kwargs:
training_id = kwargs['te_id']
else:
training_id = decoder.te_id
if retrain_flag:
raise NotImplementedError("Need to test retraining with real data")
saved_files = models.DataFile.objects.filter(entry_id=training_id)
files = {}
for fl in saved_files:
files[fl.system.name] = fl.get_path()
import bmilist
decoder = training_method(files, decoder.extractor_cls, decoder.extractor_kwargs, bmilist.kin_extractors[''], decoder.ssm,
decoder.units, update_rate=decoder.binlen, tslice=decoder.tslice, pos_key=pos_key, zscore=True, **kwargs2)
suffx = '_zscore_set_from_'+str(calc_zscore_from_te)+'_retrained'
else:
decoder.mFR = 0.
decoder.sdFR = 1.
decoder.init_zscore(mFR, sdFR)
decoder.mFR = mFR
decoder.sdFR = sdFR
suffx = '_zscore_set_from_'+str(calc_zscore_from_te)
if task_entry_id is not None:
save_new_dec(task_entry_id, decoder, suffx)
else:
return decoder, suffx
