def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) g = act.softmax(s_inst, self.W_g, self.g_strength, self.level_bias) g = np.transpose(g) l_sel = self.select_level(g) y_m = np.zeros((self.L, 1, self.M)) for l in np.arange(self.L): if l == l_sel: y_m[l, :, :], self.cumulative_memory[ l, :, :] = act.sigmoid_acc_leaky( s_trans, self.V_m[l, :, :], self.cumulative_memory[l, :, :], self.LEAK[l, 0, 0], g[l, 0]) else: self.cumulative_memory[l, :, :] *= self.LEAK[l, 0, 0] y_m[l, :, :] = act.sigmoidal(self.cumulative_memory[l, :, :]) print('\t\t\t\t MEMORY_LEVEL ', l, '\t ', y_m[l, :, :]) inp_h = np.zeros((1, self.H)) for l in np.arange(self.L): inp_h = act.linear(y_m[l, :, :], self.W_m[l, :, :]) y_h = act.sigmoidal(inp_h) Q = act.linear(y_r, self.W_r) + act.linear(y_h, self.W_h) return y_r, y_m, y_h, g, l_sel, Q
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) y_m = np.zeros((self.L, 1, self.M)) Q = act.linear(y_r, self.W_r) for l in np.arange(self.L): y_m[l,:,:], self.cumulative_memory[l,:,:] = act.sigmoid_acc(s_trans, self.V_m[l,:,:], self.cumulative_memory[l,:,:]) Q += act.linear(y_m[l,:,:], self.W_m[l,:,:]) return y_r, y_m, Q
def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) g = act.hard_sigmoid(s_inst, self.W_g, self.g_strength) y_m, self.cumulative_memory = act.sigmoid_gated( s_trans, self.V_m, self.cumulative_memory, self.memory_leak, g) Q = act.linear(y_r, self.W_r) + act.linear(y_m, self.W_m) return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) #print('Cum = ',self.cumulative_memory) if self.M != 0: y_m, self.cumulative_memory = act.sigmoid_acc( s_trans, self.V_m, self.cumulative_memory) y_tot = np.concatenate((y_r, y_m), axis=1) W_tot = np.concatenate((self.W_r, self.W_m), axis=0) Q = act.linear(y_tot, W_tot) else: y_m = None Q = act.linear(y_r, self.W_r) return y_r, y_m, Q
def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) g = act.softmax(s_inst, self.W_g, self.g_strength) g = np.transpose(g) y_m = 1e-6 * np.ones((self.L, 1, self.M)) Q = act.linear(y_r, self.W_r) for l in np.arange(self.L): y_m[l, :, :], self.memory_content[l, :, :] = act.sigmoid_acc_leaky( s_trans, self.V_m[l, :, :], self.memory_content[l, :, :], 1 - g[l, 0], g[l, 0]) Q += act.linear(y_m[l, :, :], self.W_m[l, :, :]) #print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:]) return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) if self.H_r != 0: y_h_r = act.sigmoid(y_r, self.W_r) else: y_h_r = None y_m, self.cumulative_memory = act.sigmoid_acc_leaky( s_trans, self.V_m, self.cumulative_memory, self.memory_leak) if self.H_m != 0: y_h_m = act.sigmoid(y_m, self.W_m) else: y_h_m = None if self.H_r != 0 and self.H_m != 0: y_tot = np.concatenate((y_h_r, y_h_m), axis=1) W_tot = np.concatenate((self.W_h_r, self.W_h_m), axis=0) elif self.H_r == 0 and self.H_m != 0: y_tot = np.concatenate((y_r, y_h_m), axis=1) W_tot = np.concatenate((self.W_r, self.W_h_m), axis=0) elif self.H_r != 0 and self.H_m == 0: y_tot = np.concatenate((y_h_r, y_m), axis=1) W_tot = np.concatenate((self.W_h_r, self.W_m), axis=0) else: y_tot = np.concatenate((y_r, y_m), axis=1) W_tot = np.concatenate((self.W_r, self.W_m), axis=0) #print(y_tot) Q = act.linear(y_tot, W_tot) return y_r, y_m, Q, y_h_r, y_h_m
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) y_m,self.cumulative_memory = act.sigmoid_acc_leaky(s_trans, self.V_m, self.cumulative_memory,self.memory_leak) y_tot = np.concatenate((y_r, y_m),axis=1) W_tot = np.concatenate((self.W_r, self.W_m),axis=0) Q = act.linear(y_tot, W_tot) return y_r, y_m, Q
def feedforward(self, s_inst, s_trans): g = act.hard_sigmoid(s_inst, self.W_g, self.g_strength) g = np.transpose(g) #g = np.ones((self.L,1)) y_m = 1e-6 * np.ones((self.L, 1, self.M)) inp_r = 1e-6 * np.ones((self.L, 1, self.R)) for l in np.arange(self.L): y_m[l, :, :], self.memory_content[l, :, :] = act.sigmoid_acc_leaky( s_trans, self.V_m[l, :, :], self.memory_leak, g[l, 0]) inp_r[l, :, :] = act.linear(y_m[l, :, :], self.W_m[l, :, :]) #print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:]) inp_r = np.sum(inp_r, axis=0) inp_r += act.linear(s_inst, self.V_r) y_r = act.sigmoidal(inp_r) #print(y_r) Q = act.linear(y_r, self.W_r) return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans): g_strength = 4 y_r = act.sigmoid(s_inst, self.V_r) g = act.hard_sigmoid(s_inst, self.W_g, g_strength) g = np.transpose(g) y_m = 1e-6 * np.ones((self.L, 1, self.M)) Q = act.linear(y_r, self.W_r) for l in np.arange(self.L): y_m[l, :, :], self.cumulative_memory[ l, :, :] = act.sigmoid_acc_leaky( s_trans, self.V_m[l, :, :], self.cumulative_memory[l, :, :], self.LEAK[l, 0, 0], g[l, 0]) Q += act.linear(y_m[l, :, :], self.W_m[l, :, :]) print('\t MEM STATE ', str(l), ':', y_m[l, :, :], '\t gate=', g[l, :], '\t alpha=', self.ALPHA[l, 0, 0], '\t leak=', self.LEAK[l, 0, 0]) return y_r, y_m, g, Q
def feedforward(self, s_inst, s_trans): y_r = act.sigmoid(s_inst, self.V_r) g = act.softmax(s_inst, self.W_g, self.g_strength) g = np.transpose(g) #g = np.ones((self.L,1)) y_m = np.zeros((self.L, 1, self.M)) inp_h = np.zeros((1, self.H)) for l in np.arange(self.L): y_m[l, :, :], self.cumulative_memory[ l, :, :] = act.sigmoid_acc_leaky( s_trans, self.V_m[l, :, :], self.cumulative_memory[l, :, :], self.LEAK[l, 0, 0], g[l, 0]) inp_h += act.linear(y_m[l, :, :], self.W_m[l, :, :]) #print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:]) y_h = act.sigmoidal(inp_h) Q = act.linear(y_r, self.W_r) + act.linear(y_h, self.W_h) return y_r, y_m, y_h, g, Q
def memory_gating(self, s, s_print, bias=0, gate='softmax'): if s_print != 'e': if (self.r == 0).all(): self.r = s else: #v = act.linear(s,np.transpose(self.X)) v = act.linear(s, self.X) v_i = v[np.where(s == 1)] v_j = v[np.where(self.r == 1)] if v_i == v_j: random_bin = np.random.choice(2, 1) # print('V_i: ',v_i,'\tV_j: ',v_j,'\tStoring: ',random_bin) if random_bin == 0: self.r = s if (gate == 'softmax'): random_value = np.random.random() p_storing = (np.exp(self.beta * v_i) + bias) / (np.exp( self.beta * v_i) + np.exp(self.beta * v_j) + bias) # print('V_i: ',v_i,'\tV_j: ',v_j,'\tP_storing:',p_storing) if math.isnan( p_storing ) == False: # because of the exponential and beta=12, the probability value could be a NaN if random_value <= p_storing: self.r = s else: # if the storing probability is NaN, I adopt the maximum criterion if v_i >= v_j: self.r = s elif (gate == 'max'): if v_i > v_j: self.r = s elif (gate == 'free'): self.r = s
def feedforward(self,s_inst,s_trans): y_r = act.sigmoid(s_inst, self.V_r) if self.H_r!=0: y_h_r = act.sigmoid(y_r,self.W_r) if self.perc_active!=1: max_ind = np.argsort(-np.abs(y_h_r))[0,:self.num_active_reg] y_h_r_filtered = np.zeros(np.shape(y_h_r)) y_h_r_filtered[0,max_ind] = y_h_r[0,max_ind] else: y_h_r_filtered = y_h_r else: y_h_r_filtered = None y_m,self.cumulative_memory = act.sigmoid_acc_leaky(s_trans, self.V_m, self.cumulative_memory,self.memory_leak) if self.H_m!=0: y_h_m = act.sigmoid(y_m,self.W_m) if self.perc_active!=1: max_ind = np.argsort(-np.abs(y_h_m))[0,:self.num_active_mem] y_h_m_filtered = np.zeros(np.shape(y_h_m)) y_h_m_filtered[0,max_ind] = y_h_m[0,max_ind] else: y_h_m_filtered = y_h_m else: y_h_m_filtered = None if self.H_r!=0 and self.H_m!=0: y_tot = np.concatenate((y_h_r, y_h_m),axis=1) W_tot = np.concatenate((self.W_h_r, self.W_h_m),axis=0) elif self.H_r==0 and self.H_m!=0: y_tot = np.concatenate((y_r, y_h_m),axis=1) W_tot = np.concatenate((self.W_r, self.W_h_m),axis=0) if self.H_r!=0 and self.H_m==0: y_tot = np.concatenate((y_h_r, y_m),axis=1) W_tot = np.concatenate((self.W_h_r, self.W_m),axis=0) Q = act.linear(y_tot, W_tot) return y_r, y_m, Q, y_h_r_filtered, y_h_m_filtered
f_exp = np.reshape(f,(1,self.L)) self.Tag_w_f += -self.alpha*self.Tag_w_f + np.dot(np.transpose(s_inst), f_strength*f_exp*(1-f_exp)*np.sum(np.squeeze(self.memory_content,axis=1)*np.transpose(y_m*(1-y_m),(1,0,2))*self.W_m_back[:,resp_ind,:],axis=2)) def feedforward(self,s_inst,s_trans): g_strength = 3 f_strength = 3 y_r = act.sigmoid(s_inst, self.V_r) g = act.softmax(s_inst,self.W_g, g_strength) g = np.transpose(g) f = act.hard_sigmoid(s_inst,self.W_f, f_strength) f = np.transpose(f) dwz y_m = 1e-6*np.ones((self.L,1,self.M)) Q = act.linear(y_r, self.W_r) for l in np.arange(self.L): y_m[l,:,:], self.memory_content[l,:,:] = act.sigmoid_acc_leaky(s_trans, self.V_m[l,:,:], self.memory_content[l,:,:],f[l,0],g[l,0]) Q += act.linear(y_m[l,:,:], self.W_m[l,:,:]) #print('\t MEM STATE ',str(l),':', y_m[l,:,:],'\t alpha=',self.ALPHA[l,0,0],'\t gate=',g[l,:],'\t forget=',f[l,:]) return y_r, y_m, g, f, Q ######### TRAINING + TEST FOR CPT TASKS LIKE 12AX def training(self,N_ep,p_c,conv_criterion='strong',stop=True,verbose=False): self.initialize_weights_and_tags() zero = np.zeros((1,self.S)) E = np.zeros((N))
def HER_task_AX_CPT(params_bool, params_task): from TASKS.task_AX_CPT import data_construction task = 'AX-CPT' np.random.seed(1234) #cues_vec = ['A','B','X','Y'] #pred_vec = ['LC','LW','RC','RW'] if params_task is None: N_stimuli = 40000 target_perc = 0.5 tr_perc = 0.8 else: N_stimuli = int(params_task[0]) if params_task[0] != '' else 40000 target_perc = float(params_task[1]) if params_task[1] != '' else 0.5 tr_perc = float(params_task[2]) if params_task[2] != '' else 0.8 [S_tr, O_tr, S_test, O_test, dic_stim, dic_resp] = data_construction(N=N_stimuli, perc_target=target_perc, perc_training=tr_perc, model='1') ## CONSTRUCTION OF THE HER MULTI-LEVEL NETWORK NL = 2 # number of levels (<= 3) S = np.shape(S_tr)[1] # dimension of the input P = np.shape(O_tr)[1] # dimension of the prediction vector ### Parameter values come from Table 1 of the Supplementary Material of the paper "Frontal cortex function derives from hierarchical predictive coding", W. Alexander, J. Brown learn_rate_vec = [0.075, 0.075] # learning rates beta_vec = [15, 15] # gain parameter for memory dynamics elig_decay_vec = [0.1, 0.99] # decay factors for eligibility trace bias_vec = [1, 0.1] gamma = 5 # taken from suggestion from "Extended Example of HER Model" fontTitle = 22 fontTicks = 20 verb = 0 learn_rule_WM = 'backprop' # options are: backprop or RL elig_update = 'pre' # options are: pre or post (eligibility trace update respectively at the beginning or at the end of the training iteration) if params_bool is None: do_training = True do_test = True do_weight_plots = True do_error_plots = True else: do_training = params_bool[0] do_test = params_bool[1] do_weight_plots = params_bool[2] do_error_plots = params_bool[3] error_test = False # see behavior of the network after training on specified scenarios HER = HER_arch(NL, S, P, learn_rate_vec, beta_vec, gamma, elig_decay_vec) ## TRAINING data_folder = 'HER/DATA' if do_training: HER.training(S_tr, O_tr, bias_vec, learn_rule_WM, elig_update, dic_stim, dic_resp, verb) # save trained model for l in np.arange(NL): str_err = data_folder + '/' + task + '_error.txt' #np.savetxt(str_err, E) str_mem = data_folder + '/' + task + '_weights_memory_' + str( l) + '.txt' np.savetxt(str_mem, HER.H[l].X) str_pred = data_folder + '/' + task + '_weights_prediction_' + str( l) + '.txt' np.savetxt(str_pred, HER.H[l].W) print("\nSaved model to disk.\n") else: for l in np.arange(NL): str_mem = data_folder + '/' + task + '_weights_memory_' + str( l) + '.txt' HER.H[l].X = np.loadtxt(str_mem) str_pred = data_folder + '/' + task + '_weights_prediction_' + str( l) + '.txt' HER.H[l].W = np.loadtxt(str_pred) print("\nLoaded model from disk.\n") print( '\n----------------------------------------------------\nTEST\n------------------------------------------------------\n' ) ## TEST if do_test: HER.test(S_test, O_test, dic_stim, dic_resp, verb) ## test of the error trend after training if X is presented to the system if error_test: s = np.array([[0, 0, 1, 0]]) s_prime_vec = [ np.array([[1, 0, 0, 0]]), np.array([[0, 1, 0, 0]]), np.array([[0, 0, 1, 0]]), np.array([[0, 0, 0, 1]]) ] o_vec = [ np.array([[0, 1, 1, 0]]), np.array([[1, 0, 0, 1]]), np.array([[1, 0, 0, 1]]), np.array([[1, 0, 0, 1]]) ] HER.H[0].r = s # X at base level p = act.linear(s, HER.H[0].W) print('PREDICTION VECTOR: \n', p, '\n') for s_prime, o in zip(s_prime_vec, o_vec): HER.H[1].r = s_prime # stored at upper level p_prime = act.linear(s_prime, HER.H[1].W) HER.H[0].top_down(p_prime) m = p + act.linear(s, HER.H[0].P_prime) resp_ind, p_resp = HER.H[0].compute_response(m) e, a = HER.H[0].compute_error(p, o, resp_ind) o_prime = HER.H[0].bottom_up(e) e_prime = HER.H[1].compute_error(p_prime, o_prime) print('r0: ', dic_stim[repr(s.astype(int))], '\t r1:', dic_stim[repr(s_prime.astype(int))], '\t outcome: ', dic_resp[repr(o.astype(int))], '\t response: ', dic_resp[repr(resp_ind)], '\n') print('Prediction (level 1): \n', p_prime, '\n') print('Error (level 1): \n', np.reshape(e_prime, (S, -1))) print('Modulated Prediction: ', m) print( '----------------------------------------------------------------' ) ## PLOTS # plot of the memory weights image_folder = 'HER/IMAGES' if do_weight_plots: fig1 = plt.figure(figsize=(10 * NL, 8)) for l in np.arange(NL): X = HER.H[l].X plt.subplot(1, NL, l + 1) plt.pcolor(np.flipud(X), edgecolors='k', linewidths=1) plt.set_cmap('gray_r') plt.colorbar() tit = 'MEMORY WEIGHTS: Level ' + str(l) plt.title(tit, fontweight="bold", fontsize=fontTitle) plt.xticks(np.linspace(0.5, S - 0.5, 4, endpoint=True), cues_vec, fontsize=fontTicks) plt.yticks(np.linspace(0.5, S - 0.5, 4, endpoint=True), np.flipud(cues_vec), fontsize=fontTicks) plt.show() savestr = image_folder + '/' + task + '_weights_memory.png' fig1.savefig(savestr) fig2 = plt.figure(figsize=(10 * NL, 8)) for l in np.arange(NL): W = HER.H[l].W plt.subplot(1, NL, l + 1) plt.pcolor(np.flipud(W), edgecolors='k', linewidths=1) plt.set_cmap('gray_r') plt.colorbar() tit = 'PREDICTION WEIGHTS: Level ' + str(l) plt.title(tit, fontweight="bold", fontsize=fontTitle) if l == 0: plt.xticks(np.linspace(0.5, np.shape(W)[1] - 0.5, 4, endpoint=True), pred_vec, fontsize=fontTicks) else: dx = np.shape(W)[1] / (2 * S) plt.xticks(np.linspace(dx, np.shape(W)[1] - dx, 4, endpoint=True), cues_vec, fontsize=fontTicks) plt.yticks(np.linspace(0.5, S - 0.5, 4, endpoint=True), np.flipud(cues_vec), fontsize=fontTicks) plt.show() savestr = image_folder + '/' + task + '_weights_prediction.png' fig2.savefig(savestr)