def get_allperformance(model_dirs, param_list=None): # Get all model names that match patterns (strip off .ckpt.meta at the end) model_dirs = tools.valid_model_dirs(model_dirs) final_perfs = dict() filenames = dict() if param_list is None: param_list = ['param_intsyn', 'easy_task', 'activation'] for model_dir in model_dirs: log = tools.load_log(model_dir) hp = tools.load_hp(model_dir) perf_tests = log['perf_tests'] final_perf = np.mean([float(val[-1]) for val in perf_tests.values()]) key = tuple([hp[p] for p in param_list]) if key in final_perfs.keys(): final_perfs[key].append(final_perf) else: final_perfs[key] = [final_perf] filenames[key] = model_dir for key, val in final_perfs.items(): final_perfs[key] = np.mean(val) print(key), print('{:0.3f}'.format(final_perfs[key])), print(filenames[key])
def __init__(self, model_dir): self.model_dir = model_dir self.log = tools.load_log(model_dir) self.hp = tools.load_hp(self.log['model_dir']) self.neurons = self.hp['n_rnn'] self.print_basic_info()
def plot_performanceprogress(model_dir, rule_plot=None): # Plot Training Progress log = tools.load_log(model_dir) hp = tools.load_hp(model_dir) trials = log['trials'] fs = 6 # fontsize fig = plt.figure(figsize=(3.5, 1.2)) ax = fig.add_axes([0.1, 0.25, 0.35, 0.6]) lines = list() labels = list() x_plot = np.array(trials) / 1000. if rule_plot == None: rule_plot = hp['rules'] for i, rule in enumerate(rule_plot): # line = ax1.plot(x_plot, np.log10(cost_tests[rule]),color=color_rules[i%26]) # ax2.plot(x_plot, perf_tests[rule],color=color_rules[i%26]) line = ax.plot(x_plot, np.log10(log['cost_' + rule]), color=rule_color[rule]) ax.plot(x_plot, log['perf_' + rule], color=rule_color[rule]) lines.append(line[0]) labels.append(rule_name[rule]) ax.tick_params(axis='both', which='major', labelsize=fs) ax.set_ylim([0, 1]) ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=2) ax.set_ylabel('Performance', fontsize=fs, labelpad=0) ax.locator_params(axis='x', nbins=3) ax.set_yticks([0, 1]) ax.spines["right"].set_visible(False) ax.spines["top"].set_visible(False) ax.xaxis.set_ticks_position('bottom') ax.yaxis.set_ticks_position('left') lg = fig.legend(lines, labels, title='Task', ncol=2, bbox_to_anchor=(0.47, 0.5), fontsize=fs, labelspacing=0.3, loc=6, frameon=False) plt.setp(lg.get_title(), fontsize=fs) plt.savefig('figure/Performance_Progresss.pdf', transparent=True) plt.show()
def get_n_clusters(root_dir): model_dirs = tools.valid_model_dirs(root_dir) hp_list = list() n_clusters = list() for i, model_dir in enumerate(model_dirs): if i % 50 == 0: print('Analyzing model {:d}/{:d}'.format(i, len(model_dirs))) hp = tools.load_hp(model_dir) log = tools.load_log(model_dir) # check if performance exceeds target if log['perf_min'][-1] > hp['target_perf']: n_clusters.append(log['n_cluster']) hp_list.append(hp) return n_clusters, hp_list
def __init__(self, model_dir, hp=None, sigma_rec=None, dt=None): """ Initializing the model with information from hp Args: model_dir: string, directory of the model hp: a dictionary or None sigma_rec: if not None, overwrite the sigma_rec passed by hp """ # Reset tensorflow graphs tf.reset_default_graph() # must be in the beginning if hp is None: hp = tools.load_hp(model_dir) if hp is None: raise ValueError( 'No hp found for model_dir {:s}'.format(model_dir)) tf.set_random_seed(hp['seed']) self.rng = np.random.RandomState(hp['seed']) if sigma_rec is not None: print('Overwrite sigma_rec with {:0.3f}'.format(sigma_rec)) hp['sigma_rec'] = sigma_rec if dt is not None: print('Overwrite original dt with {:0.1f}'.format(dt)) hp['dt'] = dt hp['alpha'] = 1.0 * hp['dt'] / hp['tau'] # Input, target output, and cost mask # Shape: [Time, Batch, Num_units] if hp['in_type'] != 'normal': raise ValueError('Only support in_type ' + hp['in_type']) datasetType, datasetShape = defineDatasetFormat(hp) dataset = tf.data.Dataset.from_generator( lambda: datasetGeneratorFromTaskDef(hp, 'random'), datasetType, datasetShape) dataset = dataset.prefetch(4) self.datasetTensors = dataset.make_one_shot_iterator().get_next() self._build(hp) self.model_dir = model_dir self.hp = hp
def compute_n_cluster(model_dirs): for model_dir in model_dirs: print(model_dir) log = tools.load_log(model_dir) hp = tools.load_hp(model_dir) try: analysis = clustering.Analysis(model_dir, 'rule') log['n_cluster'] = analysis.n_cluster log['model_dir'] = model_dir tools.save_log(log) except IOError: # Training never finished assert log['perf_min'][-1] <= hp['target_perf'] # analysis.plot_example_unit() # analysis.plot_variance() # analysis.plot_2Dvisualization() print("done")
def __init__(self, model_dir, hp=None, sigma_rec=None, dt=None): """ Initializing the model with information from hp Args: model_dir: string, directory of the model hp: a dictionary or None sigma_rec: if not None, overwrite the sigma_rec passed by hp """ ## Reset tensorflow graphs #tf.reset_default_graph() # must be in the beginning if hp is None: hp = tools.load_hp(model_dir) if hp is None: raise ValueError( 'No hp found for model_dir {:s}'.format(model_dir)) torch.seed(hp['seed']) self.rng = np.random.RandomState(hp['seed']) if sigma_rec is not None: print('Overwrite sigma_rec with {:0.3f}'.format(sigma_rec)) hp['sigma_rec'] = sigma_rec if dt is not None: print('Overwrite original dt with {:0.1f}'.format(dt)) hp['dt'] = dt hp['alpha'] = 1.0 * hp['dt'] / hp['tau'] # Input, target output, and cost mask # Shape: [Time, Batch, Num_units] if hp['in_type'] != 'normal': raise ValueError('Only support in_type ' + hp['in_type']) self._build(hp) self.model_dir = model_dir self.hp = hp
def get_finalperformance(model_dirs): """Get lists of final performance.""" hp = tools.load_hp(model_dirs[0]) rule_plot = hp['rules'] final_cost, final_perf = OrderedDict(), OrderedDict() for rule in rule_plot: final_cost[rule] = list() final_perf[rule] = list() training_time_plot = list() # Recording performance and cost for networks for model_dir in model_dirs: log = tools.load_log(model_dir) if log is None: continue for rule in rule_plot: final_perf[rule] += [float(log['perf_' + rule][-1])] final_cost[rule] += [float(log['cost_' + rule][-1])] training_time_plot.append(log['times'][-1]) return final_cost, final_perf, rule_plot, training_time_plot
def plot_posttrain_performance(posttrain_setup, trainables): from task import rule_name hp_target = {'posttrain_setup': posttrain_setup, 'trainables': trainables} fs = 7 fig = plt.figure(figsize=(1.5, 1.2)) ax = fig.add_axes([0.25, 0.3, 0.7, 0.65]) colors = ['xkcd:blue', 'xkcd:red'] for pretrain_setup in [1, 0]: c = colors[pretrain_setup] l = ['B', 'A'][pretrain_setup] hp_target['pretrain_setup'] = pretrain_setup model_dirs = tools.find_all_models(DATAPATH, hp_target) hp = tools.load_hp(model_dirs[0]) rule = hp['rule_trains'][0] # depends on posttrain setup for model_dir in model_dirs: log = tools.load_log(model_dir) ax.plot(np.array(log['trials']) / 1000., log['perf_' + rule], color=c, alpha=0.1) avg_perfs, trials = get_avg_performance(model_dirs, rule) l0 = ax.plot(trials / 1000., avg_perfs, color=c, label=l) ax.set_ylim([0, 1]) ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=2) ax.set_yticks([0, 1]) ax.spines["right"].set_visible(False) ax.spines["top"].set_visible(False) # lg = ax.legend(title='Pretrained set', ncol=2, loc=4, # frameon=False) plt.ylabel('Perf. of ' + rule_name[rule]) # plt.title('Training ' + hp_target['trainables']) plt.savefig('figure/Posttrain_post{:d}train{:s}.pdf'.format( posttrain_setup, trainables), transparent=True)
def plot_hist_varprop_all(model_dir, plot_control=True): ''' Plot histogram of proportion of variance for some tasks across units :param save_name: :param data_type: :param rule_pair: list of rule_pair. Show proportion of variance for the first rule :return: ''' model_dirs = tools.valid_model_dirs(model_dir) hp = tools.load_hp(model_dirs[0]) rules = hp['rules'] figsize = (7, 7) # For testing # rules, figsize = ['fdgo','reactgo','delaygo', 'fdanti', 'reactanti'], (4, 4) fs = 6 # fontsize f, axarr = plt.subplots(len(rules), len(rules), figsize=figsize) plt.subplots_adjust(left=0.1, right=0.98, bottom=0.02, top=0.9) for i in range(len(rules)): for j in range(len(rules)): ax = axarr[i, j] if i == 0: ax.set_title(rule_name[rules[j]], fontsize=fs, rotation=45, va='bottom') if j == 0: ax.set_ylabel(rule_name[rules[i]], fontsize=fs, rotation=45, ha='right') ax.spines["right"].set_visible(False) ax.spines["left"].set_visible(False) ax.spines["top"].set_visible(False) if i == j: ax.spines["bottom"].set_visible(False) ax.set_xticks([]) ax.set_yticks([]) continue hists, bins_edge = compute_hist_varprop(model_dir, (rules[i], rules[j])) hist_low, hist_med, hist_high = np.percentile(hists, [10, 50, 90], axis=0) hist_med /= hist_med.sum() # Control case if plot_control: hists_ctrl, _ = compute_hist_varprop(model_dir, (rules[i], rules[j]), random_rotation=True) _, hist_med_ctrl, _ = np.percentile(hists_ctrl, [10, 50, 90], axis=0) hist_med_ctrl /= hist_med_ctrl.sum() ax.plot((bins_edge[:-1] + bins_edge[1:]) / 2, hist_med_ctrl, color='gray', lw=0.75) ax.plot((bins_edge[:-1] + bins_edge[1:]) / 2, hist_med, color='black') plt.locator_params(nbins=3) # ax.set_ylim(bottom=-0.02*hist_med.max()) ax.set_ylim([-0.01, 0.6]) print(hist_med.max()) ax.set_xticks([-1, 1]) ax.set_xticklabels([]) if i == 0 and j == 1: ax.set_yticks([0, 0.6]) ax.spines["left"].set_visible(True) else: ax.set_yticks([]) ax.set_xlim([-1, 1]) ax.xaxis.set_ticks_position('bottom') ax.tick_params(axis='both', which='major', labelsize=fs, length=2) # plt.tight_layout() plt.savefig('figure/plot_hist_varprop_all.pdf', transparent=True)
def __init__(self, model_dir, data_type, res, normalization_method='max'): # add res by yichen hp = tools.load_hp(model_dir) # If not computed, use variance.py #fname = os.path.join(model_dir, 'variance_' + data_type + '.pkl') #res = tools.load_pickle(fname) h_var_all_ = res['h_var_all'] self.keys = res['keys'] # First only get active units. Total variance across tasks larger than 1e-3 # ind_active = np.where(h_var_all_.sum(axis=1) > 1e-2)[0] ind_active = np.where(h_var_all_.sum(axis=1) > 1e-3)[0] h_var_all = h_var_all_[ind_active, :] # Normalize by the total variance across tasks if normalization_method == 'sum': h_normvar_all = (h_var_all.T / np.sum(h_var_all, axis=1)).T elif normalization_method == 'max': h_normvar_all = (h_var_all.T / np.max(h_var_all, axis=1)).T elif normalization_method == 'none': h_normvar_all = h_var_all else: raise NotImplementedError() ################################## Clustering ################################ from sklearn import metrics X = h_normvar_all # Clustering from sklearn.cluster import AgglomerativeClustering, KMeans # Choose number of clusters that maximize silhouette score n_clusters = range(2, 30) scores = list() labels_list = list() for n_cluster in n_clusters: # clustering = AgglomerativeClustering(n_cluster, affinity='cosine', linkage='average') clustering = KMeans(n_cluster, algorithm='full', n_init=20, random_state=0) clustering.fit( X) # n_samples, n_features = n_units, n_rules/n_epochs labels = clustering.labels_ # cluster labels score = metrics.silhouette_score(X, labels) scores.append(score) labels_list.append(labels) scores = np.array(scores) # Heuristic elbow method # Choose the number of cluster when Silhouette score first falls # Choose the number of cluster when Silhouette score is maximum if data_type == 'rule': #i = np.where((scores[1:]-scores[:-1])<0)[0][0] i = np.argmax(scores) else: # The more rigorous method doesn't work well in this case i = n_clusters.index(10) labels = labels_list[i] n_cluster = n_clusters[i] print('Choosing {:d} clusters'.format(n_cluster)) # Sort clusters by its task preference (important for consistency across nets) if data_type == 'rule': label_prefs = [ np.argmax(h_normvar_all[labels == l].sum(axis=0)) for l in set(labels) ] elif data_type == 'epoch': ## TODO: this may no longer work! label_prefs = [ self.keys[np.argmax(h_normvar_all[labels == l].sum(axis=0))][0] for l in set(labels) ] ind_label_sort = np.argsort(label_prefs) label_prefs = np.array(label_prefs)[ind_label_sort] # Relabel labels2 = np.zeros_like(labels) for i, ind in enumerate(ind_label_sort): labels2[labels == ind] = i labels = labels2 # # Sort data by labels and by input connectivity # model = Model(save_name) # hp = model.hp # with tf.Session() as sess: # model.restore(sess) # var_list = sess.run(model.var_list) # # # Get connectivity # w_out = var_list[0].T # b_out = var_list[1] # w_in = var_list[2][:n_input, :].T # w_rec = var_list[2][n_input:, :].T # b_rec = var_list[3] # # # nx, nh, ny = hp['shape'] # nr = hp['n_eachring'] # # sort_by = 'w_in' # if sort_by == 'w_in': # w_in_mod1 = w_in[ind_active, :][:, 1:nr+1] # w_in_mod2 = w_in[ind_active, :][:, nr+1:2*nr+1] # w_in_modboth = w_in_mod1 + w_in_mod2 # w_prefs = np.argmax(w_in_modboth, axis=1) # elif sort_by == 'w_out': # w_prefs = np.argmax(w_out[1:, ind_active], axis=0) # # ind_sort = np.lexsort((w_prefs, labels)) # sort by labels then by prefs ###############################add by yichen############################################ #adjust the cluster order# freq = dict() for i in labels: if i in freq: freq[i] += 1 else: freq[i] = 1 tmp_seq = sorted( freq.items(), key=lambda item: item[1]) #sort by frequency ,low to high relabel_dict = dict() for i in range(len(tmp_seq)): relabel_dict[tmp_seq[i][0]] = i relabel = [] for i in labels: relabel.append(relabel_dict[i]) labels = relabel labels = np.array(labels) ######################################################################################## ind_sort = np.argsort(labels) labels = labels[ind_sort] self.h_normvar_all = h_normvar_all[ind_sort, :] self.ind_active = ind_active[ind_sort] self.n_clusters = n_clusters self.scores = scores self.n_cluster = n_cluster self.h_var_all = h_var_all self.normalization_method = normalization_method self.labels = labels self.unique_labels = np.unique(labels) self.model_dir = model_dir self.hp = hp self.data_type = data_type self.rules = hp['rules']
def _plot_performanceprogress_cont(model_dir, model_dir2=None, save=True): # Plot Training Progress log = tools.load_log(model_dir) hp = tools.load_hp(model_dir) trials = np.array(log['trials']) / 1000. times = log['times'] rule_now = log['rule_now'] if model_dir2 is not None: log2 = tools.load_log(model_dir2) trials2 = np.array(log2['trials']) / 1000. fs = 7 # fontsize lines = list() labels = list() rule_train_plot = hp['rule_trains'] rule_test_plot = hp['rules'] nx, ny = 4, 2 fig, axarr = plt.subplots(nx, ny, figsize=(3, 3), sharex=True) for i in range(int(nx * ny)): ix, iy = i % nx, int(i / nx) ax = axarr[ix, iy] if i >= len(rule_test_plot): ax.axis('off') continue rule = rule_test_plot[i] # Plot fills trials_rule_prev_end = 0 # end of previous rule training time for rule_ in rule_train_plot: if rule == rule_: ec = 'black' else: ec = (0, 0, 0, 0.1) trials_rule_now = [trials_rule_prev_end] + [ trials[ii] for ii in range(len(rule_now)) if rule_now[ii] == rule_ ] trials_rule_prev_end = trials_rule_now[-1] ax.fill_between(trials_rule_now, 0, 1, facecolor='none', edgecolor=ec, linewidth=0.5) # Plot lines line = ax.plot(trials, log['perf_' + rule], lw=1, color='gray') if model_dir2 is not None: ax.plot(trials2, log2['perf_' + rule], lw=1, color='red') lines.append(line[0]) if isinstance(rule, str): rule_name_print = rule_name[rule] else: rule_name_print = ' & '.join([rule_name[r] for r in rule]) labels.append(rule_name_print) ax.tick_params(axis='both', which='major', labelsize=fs) ax.set_ylim([0, 1.05]) ax.set_xlim([0, trials_rule_prev_end]) ax.set_yticks([0, 1]) ax.set_xticks([0, np.floor(trials_rule_prev_end / 100.) * 100]) if (ix == nx - 1) and (iy == 0): ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=1) if i == 0: ax.set_ylabel('Performance', fontsize=fs, labelpad=1) ax.spines["right"].set_visible(False) ax.spines["top"].set_visible(False) ax.xaxis.set_ticks_position('none') ax.yaxis.set_ticks_position('left') ax.set_title(rule_name[rule], fontsize=fs, y=0.87, color='black') print('Training time {:0.1f} hours'.format(times[-1] / 3600.)) if save: name = 'TrainingCont_Progress' if model_dir2 is not None: name = name + '2' plt.savefig('figure/' + name + '.pdf', transparent=True) plt.show()
def plot_histogram(): initdict = defaultdict(list) initdictother = defaultdict(list) initdictotherother = defaultdict(list) for model_dir in model_dirs: hp = tools.load_hp(model_dir) #check if performance exceeds target log = tools.load_log(model_dir) #if log['perf_avg'][-1] > hp['target_perf']: if log['perf_min'][-1] > hp['target_perf']: print('no. of clusters', log['n_cluster']) n_clusters.append(log['n_cluster']) hp_list.append(hp) initdict[hp['w_rec_init']].append(log['n_cluster']) initdict[hp['activation']].append(log['n_cluster']) #initdict[hp['rnn_type']].append(log['n_cluster']) if hp['activation'] != 'tanh': initdict[hp['rnn_type']].append(log['n_cluster']) initdictother[hp['rnn_type']+hp['activation']].append(log['n_cluster']) initdictotherother[hp['rnn_type']+hp['activation']+hp['w_rec_init']].append(log['n_cluster']) if hp['l1_h'] == 0: initdict['l1_h_0'].append(log['n_cluster']) else: #hp['l1_h'] == 1e-3 or 1e-4 or 1e-5: keyvalstr = 'l1_h_1emin'+str(int(abs(np.log10(hp['l1_h'])))) initdict[keyvalstr].append(log['n_cluster']) if hp['l1_weight'] == 0: initdict['l1_weight_0'].append(log['n_cluster']) else: #hp['l1_h'] == 1e-3 or 1e-4 or 1e-5: keyvalstr = 'l1_weight_1emin'+str(int(abs(np.log10(hp['l1_weight'])))) initdict[keyvalstr].append(log['n_cluster']) #initdict[hp['l1_weight']].append(log['n_cluster']) # Check no of clusters under various conditions. f, axarr = plt.subplots(7, 1, figsize=(3,12), sharex=True) u = 0 for key in initdict.keys(): if 'l1_' not in key: title = (key + ' ' + str(len(initdict[key])) + ' mean: '+str(round(np.mean(initdict[key]),2))) axarr[u].set_title(title) axarr[u].hist(initdict[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3) # plt.savefig('./figure/histforcases_96nets.png') # plt.savefig('./figure/histforcases__pt9_192nets.pdf') # plt.savefig('./figure/histforcases___leakygrunotanh_pt9_192nets.pdf') f, axarr = plt.subplots(4, 1, figsize=(3,8), sharex=True) u = 0 for key in initdictother.keys(): if 'l1_' not in key: axarr[u].set_title(key + ' ' + str(len(initdictother[key]))+ ' mean: '+str(round(np.mean(initdictother[key]),2)) ) axarr[u].hist(initdictother[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3) # plt.savefig('./figure/histforcases__leakyrnngrurelusoftplus_pt9_192nets.pdf') f, axarr = plt.subplots(4, 1, figsize=(3,6), sharex=True) u = 0 for key in initdictotherother.keys(): if 'l1_' not in key and 'diag' not in key: axarr[u].set_title(key + ' ' + str(len(initdictotherother[key]))+ ' mean: '+str(round(np.mean(initdictotherother[key]),2)) ) axarr[u].hist(initdictotherother[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3) # plt.savefig('./figure/histforcases_randortho_notanh_pt9_192nets.pdf') f, axarr = plt.subplots(4, 1, figsize=(3,6),sharex=True) u = 0 for key in initdictotherother.keys(): if 'l1_' not in key and 'randortho' not in key: axarr[u].set_title(key + ' ' + str(len(initdictotherother[key]))+ ' mean: '+str(round(np.mean(initdictotherother[key]),2)) ) axarr[u].hist(initdictotherother[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3) # plt.savefig('./figure/histforcases_diag_notanh_pt9_192nets.pdf') #regu-- f, axarr = plt.subplots(4, 1,figsize=(3,8),sharex=True) u = 0 for key in initdict.keys(): if 'l1_h_' in key: axarr[u].set_title(key + ' ' + str(len(initdict[key]))+ ' mean: '+str(round(np.mean(initdict[key]),2)) ) axarr[u].hist(initdict[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3) #plt.savefig('./figure/noofclusters_pt9_l1_h_192nets.pdf') f, axarr = plt.subplots(4, 1,figsize=(3,8),sharex=True) u = 0 for key in initdict.keys(): if 'l1_weight_' in key: axarr[u].set_title(key + ' ' + str(len(initdict[key])) + ' mean: '+str(round(np.mean(initdict[key]),2)) ) axarr[u].hist(initdict[key]) u += 1 f.subplots_adjust(wspace=.3, hspace=0.3)