# Copyright : (c) UC Regents, Emre Neftci
# Licence : GPLv2
#-----------------------------------------------------------------------------
from decolle.lenet_decolle_model import LenetDECOLLE
from decolle.utils import parse_args, train, test, accuracy, save_checkpoint, load_model_from_checkpoint, prepare_experiment, write_stats
import datetime, os, socket, tqdm
import numpy as np
import torch
import importlib
args = parse_args('parameters/params.yml')
device = args.device
starting_epoch = 0
params, writer, dirs = prepare_experiment(name='lenet_decolle_{0}'.format(device), args = args)
log_dir = dirs['log_dir']
checkpoint_dir = dirs['checkpoint_dir']
dataset = importlib.import_module(params['dataset'])
create_dataloader = dataset.create_dataloader
verbose = args.verbose
## Load Data
gen_train, gen_test = create_dataloader(chunk_size_train=params['chunk_size_train'],
chunk_size_test=params['chunk_size_test'],
batch_size=params['batch_size'],
dt=params['deltat'], num_workers=params['num_dl_workers'])
def main():
# fold = 'C:/Users/stanzarella/data/planned_params/FirstDefinitiveRun/'
file = 'params_MN_5cl_allcycle_cs150_ov0.5.yml'
np.set_printoptions(precision=4)
# args = parse_args('parameters/params.yml')
# args = parse_args('parameters/params_MN.yml')
args = parse_args('parameters/' + file)
# args = parse_args('parameters/params_MN_multipar.yml')
# args = parse_args('parameters/params_MN_multipar2.yml')
# args = parse_args('samples/params_to_test/0_a.yml')
device = args.device
CV = 1
for cv in range(0, CV):
starting_epoch = 0
params, writer, dirs = prepare_experiment(
name=__file__.split('/')[-1].split('.')[0], args=args)
log_dir = dirs['log_dir']
checkpoint_dir = dirs['checkpoint_dir']
dataset = importlib.import_module(params['dataset'])
try:
create_data = dataset.create_data
except AttributeError:
create_data = dataset.create_dataloader
verbose = args.verbose
## Load Data
gen_train, gen_test = create_data(
root=params['filename'],
chunk_size_train=params['chunk_size_train'],
chunk_size_test=params['chunk_size_test'],
overlap_size_train_perc=params['overlap_size_train_perc'],
overlap_size_test_perc=params['overlap_size_test_perc'],
perc_test_norm=params['perc_test_norm'],
muscle_to_exclude=params['muscle_to_exclude'],
class_to_include=params['class_to_include'],
thr_firing_excl_slice=params['thr_firing_excl_slice'],
batch_size=params['batch_size'],
dt=params['deltat'],
num_workers=params['num_dl_workers'])
# gen_train, gen_test = create_data(root=params['filename'],
# chunk_size_train=params['chunk_size_train'],
# chunk_size_test=params['chunk_size_test'],
# overlap_size_train_perc=params['overlap_size_train_perc'],
# overlap_size_test_perc=params['overlap_size_test_perc'],
# muscle_to_exclude=params['muscle_to_exclude'],
# batch_size=params['batch_size'],
# dt=params['deltat'],
# num_workers=params['num_dl_workers'])
data_batch, target_batch = next(iter(gen_train))
data_batch = torch.Tensor(data_batch).to(device)
target_batch = torch.Tensor(target_batch).to(device)
#d, t = next(iter(gen_train))
input_shape = data_batch.shape[-3:]
#Backward compatibility
if 'dropout' not in params.keys():
params['dropout'] = [.5]
if len(params['input_shape']) == 0:
params['input_shape'] = [0]
if params['input_shape'][0] == 0:
params['input_shape'] = data_batch.shape[-len(params['input_shape']
):]
if type(params['alpha']) is not list:
params['alpha'] = [params['alpha']]
if type(params['beta']) is not list:
params['beta'] = [params['beta']]
## Create Model, Optimizer and Loss
net = LenetDECOLLE1DMN(out_channels=params['out_channels'],
Nhid=np.array(params['Nhid']),
Mhid=np.array(params['Mhid']),
kernel_size=params['kernel_size'],
pool_size=params['pool_size'],
input_shape=params['input_shape'],
alpha=np.array(params['alpha']),
alpharp=np.array(params['alpharp']),
dropout=params['dropout'],
beta=np.array(params['beta']),
num_conv_layers=params['num_conv_layers'],
num_mlp_layers=params['num_mlp_layers'],
lc_ampl=params['lc_ampl'],
lif_layer_type=LIFLayer,
method=params['learning_method'],
with_output_layer=True).to(device)
if hasattr(params['learning_rate'], '__len__'):
from decolle.utils import MultiOpt
opts = []
for i in range(len(params['learning_rate'])):
opts.append(
torch.optim.Adamax(net.get_trainable_parameters(i),
lr=params['learning_rate'][i],
betas=params['betas']))
opt = MultiOpt(*opts)
else:
opt = torch.optim.Adamax(net.get_trainable_parameters(),
lr=params['learning_rate'],
betas=params['betas'])
reg_l = params['reg_l'] if 'reg_l' in params else None
if 'loss_scope' in params and params['loss_scope'] == 'crbp':
from decolle.lenet_decolle_model import CRBPLoss
loss = torch.nn.SmoothL1Loss(reduction='none')
decolle_loss = CRBPLoss(net=net, loss_fn=loss, reg_l=reg_l)
else:
loss = [torch.nn.SmoothL1Loss() for i in range(len(net))]
if net.with_output_layer:
loss[-1] = cross_entropy_one_hot
decolle_loss = DECOLLELoss(net=net, loss_fn=loss, reg_l=reg_l)
##Initialize
net.init_parameters(data_batch)
##Resume if necessary
if args.resume_from is not None:
print("Checkpoint directory " + checkpoint_dir)
print('Learning rate = {}. Resumed from checkpoint'.format(
opt.param_groups[-1]['lr']))
if not os.path.exists(checkpoint_dir) and not args.no_save:
os.makedirs(checkpoint_dir)
# Printing parameters
if args.verbose:
print('Using the following parameters:')
m = max(len(x) for x in params)
for k, v in zip(params.keys(), params.values()):
print('{}{} : {}'.format(k, ' ' * (m - len(k)), v))
print('\n------Starting training with {} DECOLLE layers-------'.format(
len(net)))
# --------TRAINING LOOP----------
if not args.no_train:
test_acc_hist = []
for e in range(starting_epoch, params['num_epochs']):
interval = e // params['lr_drop_interval']
lr = opt.param_groups[-1]['lr']
if interval > 0:
print('Changing learning rate from {} to {}'.format(
lr, opt.param_groups[-1]['lr']))
opt.param_groups[-1]['lr'] = np.array(
params['learning_rate']) / (interval *
params['lr_drop_factor'])
else:
print('Changing learning rate from {} to {}'.format(
lr, opt.param_groups[-1]['lr']))
opt.param_groups[-1]['lr'] = np.array(
params['learning_rate'])
if (e % params['test_interval']) == 0 and e != 0:
print('---------------Epoch {}-------------'.format(e))
if not args.no_save:
print('---------Saving checkpoint---------')
save_checkpoint(e, checkpoint_dir, net, opt)
test_loss, test_acc = test(gen_test,
decolle_loss,
net,
params['burnin_steps'],
print_error=True)
test_acc_hist.append(test_acc)
if not args.no_save:
write_stats(e, test_acc, test_loss, writer)
np.save(
log_dir + '/test_acc.npy',
np.array(test_acc_hist),
)
total_loss, act_rate = train(
gen_train,
decolle_loss,
net,
opt,
e,
params['burnin_steps'],
online_update=params['online_update'])
if not args.no_save:
for i in range(len(net)):
writer.add_scalar('/act_rate/{0}'.format(i),
act_rate[i], e)
writer.add_scalar('/total_loss/{0}'.format(i),
total_loss[i], e)
starting_epoch = load_model_from_checkpoint(
checkpoint_dir, net, opt)
net.err_enc_layers[i].last_err_rate for i in range(len(net))
]))
total_loss /= t_sample
print('Loss {0}'.format(total_loss))
print('Activity Rate {0}'.format(act_rate))
return total_loss, act_rate, error_rates
np.set_printoptions(precision=4)
args = parse_args('parameters/params.yml')
device = args.device
starting_epoch = 0
params, writer, dirs = prepare_experiment(
name=__file__.split('/')[-1].split('.')[0], args=args)
log_dir = dirs['log_dir']
checkpoint_dir = dirs['checkpoint_dir']
dataset = importlib.import_module(params['dataset'])
try:
create_data = dataset.create_data
except AttributeError:
create_data = dataset.create_dataloader
verbose = args.verbose
## Load Data
gen_train, gen_test = create_data(chunk_size_train=params['chunk_size_train'],
chunk_size_test=params['chunk_size_test'],
batch_size=params['batch_size'],
def main():
np.set_printoptions(precision=4)
# args = parse_args('parameters/params.yml')
args = parse_args('parameters/params_MN.yml')
# args = parse_args('parameters/params_MN_multipar.yml')
# args = parse_args('parameters/params_MN_multipar2.yml')
# args = parse_args('samples/params_to_test/0_a.yml')
device = args.device
path_to_save='C:/Users/stanzarella/OneDrive - Fondazione Istituto Italiano Tecnologia/Documents/IIT_Project/2.Codes/FromLiterature/decolle-public/scripts/stats/'
starting_epoch = 0
params, writer, dirs = prepare_experiment(name=__file__.split('/')[-1].split('.')[0], args=args)
# log_dir = dirs['log_dir']
# checkpoint_dir = dirs['checkpoint_dir']
dataset = importlib.import_module(params['dataset'])
try:
create_data = dataset.create_data
except AttributeError:
create_data = dataset.create_dataloader
verbose = args.verbose
## Load Data
gen_train, gen_test = create_data(root=params['filename'],
chunk_size_train=params['chunk_size_train'],
chunk_size_test=params['chunk_size_test'],
overlap_size_train_perc=params['overlap_size_train_perc'],
overlap_size_test_perc=params['overlap_size_test_perc'],
perc_test_norm=params['perc_test_norm'],
muscle_to_exclude=params['muscle_to_exclude'],
class_to_include=params['class_to_include'],
thr_firing_excl_slice=params['thr_firing_excl_slice'],
batch_size=params['batch_size'],
dt=params['deltat'],
num_workers=params['num_dl_workers'])
# data_batch, target_batch = next(iter(gen_train))
# print(data_batch)
# print(target_batch)
kernel = 'linear' # 'rbf'
Sv = []
Dv = []
Tv = []
TrAccS = []
TrAccD = []
# S = np.zeros([data_batch.size()[3], data_batch.size()[0]])
# D = np.zeros([data_batch.size()[3], data_batch.size()[0]])
# T = np.zeros([data_batch.size()[3], data_batch.size()[0]])
for e in range(starting_epoch, params['num_epochs']):
# Train
S_epo = []
D_epo = []
T_epo = []
TrAccS_epo = []
TrAccD_epo = []
TrAccS_epo_test = []
TrAccD_epo_test = []
TrAccS_epo_test_cum = []
TrAccD_epo_test_cum = []
X_S_cum = []
# X_D_cum = []
Y_cum = []
M = 0
iter_gen_train = iter(gen_train)
for data_batch, target_batch in tqdm.tqdm(iter_gen_train, desc='Epoch {}'.format(e)):
M = np.max([data_batch.size()[0], M])
S = np.zeros([data_batch.size()[2], M])
D = np.zeros([data_batch.size()[2], M])
T = np.zeros([data_batch.size()[2], M])
X_S = []
X_D = []
Y = []
for jjj in range(0,data_batch.size()[0]): # Batches
xs = []
xd = []
for iii in range(0, data_batch.size()[2]): # Neurons
S[iii,jjj] = int(data_batch[jjj,:,iii].sum())
xs.append(S[iii,jjj])
if S[iii,jjj] > 0 :
D[iii, jjj] = int( np.diff(np.where(data_batch[jjj, :, iii]==1)).sum()/S[iii,jjj] )
xd.append(D[iii, jjj])
else:
xd.append(0)
T[iii, jjj] = int(np.array(np.where(target_batch[jjj,iii,:]==1)))
X_S.append(xs)
X_D.append(xd)
Y.append(T[0,jjj])
X_S_cum.append(xs)
# X_D_cum.append(xd)
Y_cum.append(T[0, jjj])
# SVM ---------------------------------------------------------------------------------------------------
# Training only with this batch
# Trained with spike counting
clf_S = svm.SVC(kernel=kernel, C=1000)
clf_S.fit(X_S, Y)
y_pred_S = clf_S.predict(X_S)
train_acc_S = metrics.accuracy_score(Y, y_pred_S)
# Trained with ISI
clf_D = svm.SVC(kernel=kernel, C=1000)
clf_D.fit(X_D, Y)
y_pred_D = clf_D.predict(X_D)
train_acc_D = metrics.accuracy_score(Y, y_pred_D)
# Training with this batch and the previous
# Trained with spike counting
clf_S_cum = svm.SVC(kernel=kernel, C=1000)
clf_S_cum.fit(X_S_cum, Y_cum)
# y_pred_S = clf_S.predict(X_S)
# train_acc_S = metrics.accuracy_score(Y, y_pred_S)
# Trained with ISI
# clf_D_cum = svm.SVC(kernel='linear', C=1000)
# clf_D_cum.fit(X_D_cum, Y_cum)
# y_pred_D = clf_D.predict(X_D)
# train_acc_D = metrics.accuracy_score(Y, y_pred_D)
S_epo.append(S)
D_epo.append(D)
T_epo.append(T)
TrAccS_epo.append(train_acc_S)
TrAccD_epo.append(train_acc_D)
## Test per each train batch with all test batches
train_acc_S_test = []
train_acc_D_test = []
train_acc_S_test_cum = []
train_acc_D_test_cum = []
iter_gen_test = iter(gen_test)
for data_batch_test, target_batch_test in tqdm.tqdm(iter_gen_test, desc='Testing'):
S_test = []
D_test = []
Y_test = []
for jjj in range(0, data_batch_test.size()[0]): # Batches
xs = []
xd = []
for iii in range(0, data_batch_test.size()[2]): # Neurons
ss = int(data_batch_test[jjj,:,iii].sum())
xs.append(ss)
if ss > 0:
dd = int(np.diff(np.where(data_batch_test[jjj, :, iii] == 1)).sum() / ss)
xd.append(dd)
else:
xd.append(0)
tt = int(np.array(np.where(target_batch_test[jjj, iii, :] == 1)))
S_test.append(xs)
D_test.append(xd)
Y_test.append(tt)
# Tested with spike counting
y_pred_S_test = clf_S.predict(S_test)
y_pred_S_test_cum = clf_S_cum.predict(S_test)
if y_pred_S_test.__len__() == Y_test.__len__():
train_acc_S_test.append(metrics.accuracy_score(Y_test, y_pred_S_test))
if y_pred_S_test_cum.__len__() == Y_test.__len__():
train_acc_S_test_cum.append(metrics.accuracy_score(Y_test, y_pred_S_test_cum))
# Tested with ISI
y_pred_D_test = clf_D.predict(D_test)
# y_pred_D_test_cum = clf_D_cum.predict(D_test)
if y_pred_D_test.__len__() == Y_test.__len__():
train_acc_D_test.append(metrics.accuracy_score(Y_test, y_pred_D_test))
# if y_pred_D_test_cum.__len__() == Y_test.__len__():
# train_acc_D_test_cum.append(metrics.accuracy_score(Y_test, y_pred_D_test_cum))
TrAccS_epo_test.append(train_acc_S_test)
TrAccD_epo_test.append(train_acc_D_test)
TrAccS_epo_test_cum.append(train_acc_S_test_cum)
TrAccD_epo_test_cum.append(train_acc_D_test_cum)
# SAVE
struct = {'S': S_epo, 'D': D_epo, 'T': T_epo,'TrAccS':TrAccS_epo,'TrAccD':TrAccD_epo,'TrAccS_test':TrAccS_epo_test,'TrAccS_test_cum':TrAccS_epo_test_cum,'TrAccD_test':TrAccD_epo_test,'Params': params}
spio.savemat(path_to_save + 'matlab_matrix_' + str(e) + '_'+ kernel.upper() +'.mat', struct)
