def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
g = act.softmax(s_inst, self.W_g, self.g_strength, self.level_bias)
g = np.transpose(g)
l_sel = self.select_level(g)
y_m = np.zeros((self.L, 1, self.M))
for l in np.arange(self.L):
if l == l_sel:
y_m[l, :, :], self.cumulative_memory[
l, :, :] = act.sigmoid_acc_leaky(
s_trans, self.V_m[l, :, :],
self.cumulative_memory[l, :, :], self.LEAK[l, 0,
0], g[l, 0])
else:
self.cumulative_memory[l, :, :] *= self.LEAK[l, 0, 0]
y_m[l, :, :] = act.sigmoidal(self.cumulative_memory[l, :, :])
print('\t\t\t\t MEMORY_LEVEL ', l, '\t ', y_m[l, :, :])
inp_h = np.zeros((1, self.H))
for l in np.arange(self.L):
inp_h = act.linear(y_m[l, :, :], self.W_m[l, :, :])
y_h = act.sigmoidal(inp_h)
Q = act.linear(y_r, self.W_r) + act.linear(y_h, self.W_h)
return y_r, y_m, y_h, g, l_sel, Q
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) y_m = np.zeros((self.L, 1, self.M)) Q = act.linear(y_r, self.W_r) for l in np.arange(self.L): y_m[l,:,:], self.cumulative_memory[l,:,:] = act.sigmoid_acc(s_trans, self.V_m[l,:,:], self.cumulative_memory[l,:,:]) Q += act.linear(y_m[l,:,:], self.W_m[l,:,:]) return y_r, y_m, Q
def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
g = act.hard_sigmoid(s_inst, self.W_g, self.g_strength)
y_m, self.cumulative_memory = act.sigmoid_gated(
s_trans, self.V_m, self.cumulative_memory, self.memory_leak, g)
Q = act.linear(y_r, self.W_r) + act.linear(y_m, self.W_m)
return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
#print('Cum = ',self.cumulative_memory)
if self.M != 0:
y_m, self.cumulative_memory = act.sigmoid_acc(
s_trans, self.V_m, self.cumulative_memory)
y_tot = np.concatenate((y_r, y_m), axis=1)
W_tot = np.concatenate((self.W_r, self.W_m), axis=0)
Q = act.linear(y_tot, W_tot)
else:
y_m = None
Q = act.linear(y_r, self.W_r)
return y_r, y_m, Q
def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
g = act.softmax(s_inst, self.W_g, self.g_strength)
g = np.transpose(g)
y_m = 1e-6 * np.ones((self.L, 1, self.M))
Q = act.linear(y_r, self.W_r)
for l in np.arange(self.L):
y_m[l, :, :], self.memory_content[l, :, :] = act.sigmoid_acc_leaky(
s_trans, self.V_m[l, :, :], self.memory_content[l, :, :],
1 - g[l, 0], g[l, 0])
Q += act.linear(y_m[l, :, :], self.W_m[l, :, :])
#print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:])
return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
if self.H_r != 0:
y_h_r = act.sigmoid(y_r, self.W_r)
else:
y_h_r = None
y_m, self.cumulative_memory = act.sigmoid_acc_leaky(
s_trans, self.V_m, self.cumulative_memory, self.memory_leak)
if self.H_m != 0:
y_h_m = act.sigmoid(y_m, self.W_m)
else:
y_h_m = None
if self.H_r != 0 and self.H_m != 0:
y_tot = np.concatenate((y_h_r, y_h_m), axis=1)
W_tot = np.concatenate((self.W_h_r, self.W_h_m), axis=0)
elif self.H_r == 0 and self.H_m != 0:
y_tot = np.concatenate((y_r, y_h_m), axis=1)
W_tot = np.concatenate((self.W_r, self.W_h_m), axis=0)
elif self.H_r != 0 and self.H_m == 0:
y_tot = np.concatenate((y_h_r, y_m), axis=1)
W_tot = np.concatenate((self.W_h_r, self.W_m), axis=0)
else:
y_tot = np.concatenate((y_r, y_m), axis=1)
W_tot = np.concatenate((self.W_r, self.W_m), axis=0)
#print(y_tot)
Q = act.linear(y_tot, W_tot)
return y_r, y_m, Q, y_h_r, y_h_m
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) y_m,self.cumulative_memory = act.sigmoid_acc_leaky(s_trans, self.V_m, self.cumulative_memory,self.memory_leak) y_tot = np.concatenate((y_r, y_m),axis=1) W_tot = np.concatenate((self.W_r, self.W_m),axis=0) Q = act.linear(y_tot, W_tot) return y_r, y_m, Q
def feedforward(self, s_inst, s_trans):
g = act.hard_sigmoid(s_inst, self.W_g, self.g_strength)
g = np.transpose(g)
#g = np.ones((self.L,1))
y_m = 1e-6 * np.ones((self.L, 1, self.M))
inp_r = 1e-6 * np.ones((self.L, 1, self.R))
for l in np.arange(self.L):
y_m[l, :, :], self.memory_content[l, :, :] = act.sigmoid_acc_leaky(
s_trans, self.V_m[l, :, :], self.memory_leak, g[l, 0])
inp_r[l, :, :] = act.linear(y_m[l, :, :], self.W_m[l, :, :])
#print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:])
inp_r = np.sum(inp_r, axis=0)
inp_r += act.linear(s_inst, self.V_r)
y_r = act.sigmoidal(inp_r)
#print(y_r)
Q = act.linear(y_r, self.W_r)
return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans):
g_strength = 4
y_r = act.sigmoid(s_inst, self.V_r)
g = act.hard_sigmoid(s_inst, self.W_g, g_strength)
g = np.transpose(g)
y_m = 1e-6 * np.ones((self.L, 1, self.M))
Q = act.linear(y_r, self.W_r)
for l in np.arange(self.L):
y_m[l, :, :], self.cumulative_memory[
l, :, :] = act.sigmoid_acc_leaky(
s_trans, self.V_m[l, :, :],
self.cumulative_memory[l, :, :], self.LEAK[l, 0, 0], g[l,
0])
Q += act.linear(y_m[l, :, :], self.W_m[l, :, :])
print('\t MEM STATE ', str(l), ':', y_m[l, :, :], '\t gate=',
g[l, :], '\t alpha=', self.ALPHA[l, 0, 0], '\t leak=',
self.LEAK[l, 0, 0])
return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans):
y_r = act.sigmoid(s_inst, self.V_r)
g = act.softmax(s_inst, self.W_g, self.g_strength)
g = np.transpose(g)
#g = np.ones((self.L,1))
y_m = np.zeros((self.L, 1, self.M))
inp_h = np.zeros((1, self.H))
for l in np.arange(self.L):
y_m[l, :, :], self.cumulative_memory[
l, :, :] = act.sigmoid_acc_leaky(
s_trans, self.V_m[l, :, :],
self.cumulative_memory[l, :, :], self.LEAK[l, 0, 0], g[l,
0])
inp_h += act.linear(y_m[l, :, :], self.W_m[l, :, :])
#print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:])
y_h = act.sigmoidal(inp_h)
Q = act.linear(y_r, self.W_r) + act.linear(y_h, self.W_h)
return y_r, y_m, y_h, g, Q
def memory_gating(self, s, s_print, bias=0, gate='softmax'):
if s_print != 'e':
if (self.r == 0).all():
self.r = s
else:
#v = act.linear(s,np.transpose(self.X))
v = act.linear(s, self.X)
v_i = v[np.where(s == 1)]
v_j = v[np.where(self.r == 1)]
if v_i == v_j:
random_bin = np.random.choice(2, 1)
# print('V_i: ',v_i,'\tV_j: ',v_j,'\tStoring: ',random_bin)
if random_bin == 0:
self.r = s
if (gate == 'softmax'):
random_value = np.random.random()
p_storing = (np.exp(self.beta * v_i) + bias) / (np.exp(
self.beta * v_i) + np.exp(self.beta * v_j) + bias)
# print('V_i: ',v_i,'\tV_j: ',v_j,'\tP_storing:',p_storing)
if math.isnan(
p_storing
) == False: # because of the exponential and beta=12, the probability value could be a NaN
if random_value <= p_storing:
self.r = s
else:
# if the storing probability is NaN, I adopt the maximum criterion
if v_i >= v_j:
self.r = s
elif (gate == 'max'):
if v_i > v_j:
self.r = s
elif (gate == 'free'):
self.r = s
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) if self.H_r!=0: y_h_r = act.sigmoid(y_r,self.W_r) if self.perc_active!=1: max_ind = np.argsort(-np.abs(y_h_r))[0,:self.num_active_reg] y_h_r_filtered = np.zeros(np.shape(y_h_r)) y_h_r_filtered[0,max_ind] = y_h_r[0,max_ind] else: y_h_r_filtered = y_h_r else: y_h_r_filtered = None y_m,self.cumulative_memory = act.sigmoid_acc_leaky(s_trans, self.V_m, self.cumulative_memory,self.memory_leak) if self.H_m!=0: y_h_m = act.sigmoid(y_m,self.W_m) if self.perc_active!=1: max_ind = np.argsort(-np.abs(y_h_m))[0,:self.num_active_mem] y_h_m_filtered = np.zeros(np.shape(y_h_m)) y_h_m_filtered[0,max_ind] = y_h_m[0,max_ind] else: y_h_m_filtered = y_h_m else: y_h_m_filtered = None if self.H_r!=0 and self.H_m!=0: y_tot = np.concatenate((y_h_r, y_h_m),axis=1) W_tot = np.concatenate((self.W_h_r, self.W_h_m),axis=0) elif self.H_r==0 and self.H_m!=0: y_tot = np.concatenate((y_r, y_h_m),axis=1) W_tot = np.concatenate((self.W_r, self.W_h_m),axis=0) if self.H_r!=0 and self.H_m==0: y_tot = np.concatenate((y_h_r, y_m),axis=1) W_tot = np.concatenate((self.W_h_r, self.W_m),axis=0) Q = act.linear(y_tot, W_tot) return y_r, y_m, Q, y_h_r_filtered, y_h_m_filtered
f_exp = np.reshape(f,(1,self.L))
self.Tag_w_f += -self.alpha*self.Tag_w_f + np.dot(np.transpose(s_inst), f_strength*f_exp*(1-f_exp)*np.sum(np.squeeze(self.memory_content,axis=1)*np.transpose(y_m*(1-y_m),(1,0,2))*self.W_m_back[:,resp_ind,:],axis=2))
def feedforward(self,s_inst,s_trans):
g_strength = 3
f_strength = 3
y_r = act.sigmoid(s_inst, self.V_r)
g = act.softmax(s_inst,self.W_g, g_strength)
g = np.transpose(g)
f = act.hard_sigmoid(s_inst,self.W_f, f_strength)
f = np.transpose(f)
dwz
y_m = 1e-6*np.ones((self.L,1,self.M))
Q = act.linear(y_r, self.W_r)
for l in np.arange(self.L):
y_m[l,:,:], self.memory_content[l,:,:] = act.sigmoid_acc_leaky(s_trans, self.V_m[l,:,:], self.memory_content[l,:,:],f[l,0],g[l,0])
Q += act.linear(y_m[l,:,:], self.W_m[l,:,:])
#print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:])
return y_r, y_m, g, f, Q
######### TRAINING + TEST FOR CPT TASKS LIKE 12AX
def training(self,N_ep,p_c,conv_criterion='strong',stop=True,verbose=False):
self.initialize_weights_and_tags()
zero = np.zeros((1,self.S))
E = np.zeros((N))
def HER_task_AX_CPT(params_bool, params_task):
from TASKS.task_AX_CPT import data_construction
task = 'AX-CPT'
np.random.seed(1234)
#cues_vec = ['A','B','X','Y']
#pred_vec = ['LC','LW','RC','RW']
if params_task is None:
N_stimuli = 40000
target_perc = 0.5
tr_perc = 0.8
else:
N_stimuli = int(params_task[0]) if params_task[0] != '' else 40000
target_perc = float(params_task[1]) if params_task[1] != '' else 0.5
tr_perc = float(params_task[2]) if params_task[2] != '' else 0.8
[S_tr, O_tr, S_test, O_test, dic_stim,
dic_resp] = data_construction(N=N_stimuli,
perc_target=target_perc,
perc_training=tr_perc,
model='1')
## CONSTRUCTION OF THE HER MULTI-LEVEL NETWORK
NL = 2 # number of levels (<= 3)
S = np.shape(S_tr)[1] # dimension of the input
P = np.shape(O_tr)[1] # dimension of the prediction vector
### Parameter values come from Table 1 of the Supplementary Material of the paper "Frontal cortex function derives from hierarchical predictive coding", W. Alexander, J. Brown
learn_rate_vec = [0.075, 0.075] # learning rates
beta_vec = [15, 15] # gain parameter for memory dynamics
elig_decay_vec = [0.1, 0.99] # decay factors for eligibility trace
bias_vec = [1, 0.1]
gamma = 5 # taken from suggestion from "Extended Example of HER Model"
fontTitle = 22
fontTicks = 20
verb = 0
learn_rule_WM = 'backprop' # options are: backprop or RL
elig_update = 'pre' # options are: pre or post (eligibility trace update respectively at the beginning or at the end of the training iteration)
if params_bool is None:
do_training = True
do_test = True
do_weight_plots = True
do_error_plots = True
else:
do_training = params_bool[0]
do_test = params_bool[1]
do_weight_plots = params_bool[2]
do_error_plots = params_bool[3]
error_test = False # see behavior of the network after training on specified scenarios
HER = HER_arch(NL, S, P, learn_rate_vec, beta_vec, gamma, elig_decay_vec)
## TRAINING
data_folder = 'HER/DATA'
if do_training:
HER.training(S_tr, O_tr, bias_vec, learn_rule_WM, elig_update,
dic_stim, dic_resp, verb)
# save trained model
for l in np.arange(NL):
str_err = data_folder + '/' + task + '_error.txt'
#np.savetxt(str_err, E)
str_mem = data_folder + '/' + task + '_weights_memory_' + str(
l) + '.txt'
np.savetxt(str_mem, HER.H[l].X)
str_pred = data_folder + '/' + task + '_weights_prediction_' + str(
l) + '.txt'
np.savetxt(str_pred, HER.H[l].W)
print("\nSaved model to disk.\n")
else:
for l in np.arange(NL):
str_mem = data_folder + '/' + task + '_weights_memory_' + str(
l) + '.txt'
HER.H[l].X = np.loadtxt(str_mem)
str_pred = data_folder + '/' + task + '_weights_prediction_' + str(
l) + '.txt'
HER.H[l].W = np.loadtxt(str_pred)
print("\nLoaded model from disk.\n")
print(
'\n----------------------------------------------------\nTEST\n------------------------------------------------------\n'
)
## TEST
if do_test:
HER.test(S_test, O_test, dic_stim, dic_resp, verb)
## test of the error trend after training if X is presented to the system
if error_test:
s = np.array([[0, 0, 1, 0]])
s_prime_vec = [
np.array([[1, 0, 0, 0]]),
np.array([[0, 1, 0, 0]]),
np.array([[0, 0, 1, 0]]),
np.array([[0, 0, 0, 1]])
]
o_vec = [
np.array([[0, 1, 1, 0]]),
np.array([[1, 0, 0, 1]]),
np.array([[1, 0, 0, 1]]),
np.array([[1, 0, 0, 1]])
]
HER.H[0].r = s # X at base level
p = act.linear(s, HER.H[0].W)
print('PREDICTION VECTOR: \n', p, '\n')
for s_prime, o in zip(s_prime_vec, o_vec):
HER.H[1].r = s_prime # stored at upper level
p_prime = act.linear(s_prime, HER.H[1].W)
HER.H[0].top_down(p_prime)
m = p + act.linear(s, HER.H[0].P_prime)
resp_ind, p_resp = HER.H[0].compute_response(m)
e, a = HER.H[0].compute_error(p, o, resp_ind)
o_prime = HER.H[0].bottom_up(e)
e_prime = HER.H[1].compute_error(p_prime, o_prime)
print('r0: ', dic_stim[repr(s.astype(int))], '\t r1:',
dic_stim[repr(s_prime.astype(int))], '\t outcome: ',
dic_resp[repr(o.astype(int))], '\t response: ',
dic_resp[repr(resp_ind)], '\n')
print('Prediction (level 1): \n', p_prime, '\n')
print('Error (level 1): \n', np.reshape(e_prime, (S, -1)))
print('Modulated Prediction: ', m)
print(
'----------------------------------------------------------------'
)
## PLOTS
# plot of the memory weights
image_folder = 'HER/IMAGES'
if do_weight_plots:
fig1 = plt.figure(figsize=(10 * NL, 8))
for l in np.arange(NL):
X = HER.H[l].X
plt.subplot(1, NL, l + 1)
plt.pcolor(np.flipud(X), edgecolors='k', linewidths=1)
plt.set_cmap('gray_r')
plt.colorbar()
tit = 'MEMORY WEIGHTS: Level ' + str(l)
plt.title(tit, fontweight="bold", fontsize=fontTitle)
plt.xticks(np.linspace(0.5, S - 0.5, 4, endpoint=True),
cues_vec,
fontsize=fontTicks)
plt.yticks(np.linspace(0.5, S - 0.5, 4, endpoint=True),
np.flipud(cues_vec),
fontsize=fontTicks)
plt.show()
savestr = image_folder + '/' + task + '_weights_memory.png'
fig1.savefig(savestr)
fig2 = plt.figure(figsize=(10 * NL, 8))
for l in np.arange(NL):
W = HER.H[l].W
plt.subplot(1, NL, l + 1)
plt.pcolor(np.flipud(W), edgecolors='k', linewidths=1)
plt.set_cmap('gray_r')
plt.colorbar()
tit = 'PREDICTION WEIGHTS: Level ' + str(l)
plt.title(tit, fontweight="bold", fontsize=fontTitle)
if l == 0:
plt.xticks(np.linspace(0.5,
np.shape(W)[1] - 0.5,
4,
endpoint=True),
pred_vec,
fontsize=fontTicks)
else:
dx = np.shape(W)[1] / (2 * S)
plt.xticks(np.linspace(dx,
np.shape(W)[1] - dx,
4,
endpoint=True),
cues_vec,
fontsize=fontTicks)
plt.yticks(np.linspace(0.5, S - 0.5, 4, endpoint=True),
np.flipud(cues_vec),
fontsize=fontTicks)
plt.show()
savestr = image_folder + '/' + task + '_weights_prediction.png'
fig2.savefig(savestr)