def get_allperformance(model_dirs, param_list=None):
# Get all model names that match patterns (strip off .ckpt.meta at the end)
model_dirs = tools.valid_model_dirs(model_dirs)
final_perfs = dict()
filenames = dict()
if param_list is None:
param_list = ['param_intsyn', 'easy_task', 'activation']
for model_dir in model_dirs:
log = tools.load_log(model_dir)
hp = tools.load_hp(model_dir)
perf_tests = log['perf_tests']
final_perf = np.mean([float(val[-1]) for val in perf_tests.values()])
key = tuple([hp[p] for p in param_list])
if key in final_perfs.keys():
final_perfs[key].append(final_perf)
else:
final_perfs[key] = [final_perf]
filenames[key] = model_dir
for key, val in final_perfs.items():
final_perfs[key] = np.mean(val)
print(key),
print('{:0.3f}'.format(final_perfs[key])),
print(filenames[key])
def __init__(self, model_dir):
self.model_dir = model_dir
self.log = tools.load_log(model_dir)
self.hp = tools.load_hp(self.log['model_dir'])
self.neurons = self.hp['n_rnn']
self.print_basic_info()
def plot_performanceprogress(model_dir, rule_plot=None):
# Plot Training Progress
log = tools.load_log(model_dir)
hp = tools.load_hp(model_dir)
trials = log['trials']
fs = 6 # fontsize
fig = plt.figure(figsize=(3.5, 1.2))
ax = fig.add_axes([0.1, 0.25, 0.35, 0.6])
lines = list()
labels = list()
x_plot = np.array(trials) / 1000.
if rule_plot == None:
rule_plot = hp['rules']
for i, rule in enumerate(rule_plot):
# line = ax1.plot(x_plot, np.log10(cost_tests[rule]),color=color_rules[i%26])
# ax2.plot(x_plot, perf_tests[rule],color=color_rules[i%26])
line = ax.plot(x_plot,
np.log10(log['cost_' + rule]),
color=rule_color[rule])
ax.plot(x_plot, log['perf_' + rule], color=rule_color[rule])
lines.append(line[0])
labels.append(rule_name[rule])
ax.tick_params(axis='both', which='major', labelsize=fs)
ax.set_ylim([0, 1])
ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=2)
ax.set_ylabel('Performance', fontsize=fs, labelpad=0)
ax.locator_params(axis='x', nbins=3)
ax.set_yticks([0, 1])
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
lg = fig.legend(lines,
labels,
title='Task',
ncol=2,
bbox_to_anchor=(0.47, 0.5),
fontsize=fs,
labelspacing=0.3,
loc=6,
frameon=False)
plt.setp(lg.get_title(), fontsize=fs)
plt.savefig('figure/Performance_Progresss.pdf', transparent=True)
plt.show()
def get_n_clusters(root_dir):
model_dirs = tools.valid_model_dirs(root_dir)
hp_list = list()
n_clusters = list()
for i, model_dir in enumerate(model_dirs):
if i % 50 == 0:
print('Analyzing model {:d}/{:d}'.format(i, len(model_dirs)))
hp = tools.load_hp(model_dir)
log = tools.load_log(model_dir)
# check if performance exceeds target
if log['perf_min'][-1] > hp['target_perf']:
n_clusters.append(log['n_cluster'])
hp_list.append(hp)
return n_clusters, hp_list
def __init__(self, model_dir, hp=None, sigma_rec=None, dt=None):
"""
Initializing the model with information from hp
Args:
model_dir: string, directory of the model
hp: a dictionary or None
sigma_rec: if not None, overwrite the sigma_rec passed by hp
"""
# Reset tensorflow graphs
tf.reset_default_graph() # must be in the beginning
if hp is None:
hp = tools.load_hp(model_dir)
if hp is None:
raise ValueError(
'No hp found for model_dir {:s}'.format(model_dir))
tf.set_random_seed(hp['seed'])
self.rng = np.random.RandomState(hp['seed'])
if sigma_rec is not None:
print('Overwrite sigma_rec with {:0.3f}'.format(sigma_rec))
hp['sigma_rec'] = sigma_rec
if dt is not None:
print('Overwrite original dt with {:0.1f}'.format(dt))
hp['dt'] = dt
hp['alpha'] = 1.0 * hp['dt'] / hp['tau']
# Input, target output, and cost mask
# Shape: [Time, Batch, Num_units]
if hp['in_type'] != 'normal':
raise ValueError('Only support in_type ' + hp['in_type'])
datasetType, datasetShape = defineDatasetFormat(hp)
dataset = tf.data.Dataset.from_generator(
lambda: datasetGeneratorFromTaskDef(hp, 'random'), datasetType,
datasetShape)
dataset = dataset.prefetch(4)
self.datasetTensors = dataset.make_one_shot_iterator().get_next()
self._build(hp)
self.model_dir = model_dir
self.hp = hp
def compute_n_cluster(model_dirs):
for model_dir in model_dirs:
print(model_dir)
log = tools.load_log(model_dir)
hp = tools.load_hp(model_dir)
try:
analysis = clustering.Analysis(model_dir, 'rule')
log['n_cluster'] = analysis.n_cluster
log['model_dir'] = model_dir
tools.save_log(log)
except IOError:
# Training never finished
assert log['perf_min'][-1] <= hp['target_perf']
# analysis.plot_example_unit()
# analysis.plot_variance()
# analysis.plot_2Dvisualization()
print("done")
def __init__(self, model_dir, hp=None, sigma_rec=None, dt=None):
"""
Initializing the model with information from hp
Args:
model_dir: string, directory of the model
hp: a dictionary or None
sigma_rec: if not None, overwrite the sigma_rec passed by hp
"""
## Reset tensorflow graphs
#tf.reset_default_graph() # must be in the beginning
if hp is None:
hp = tools.load_hp(model_dir)
if hp is None:
raise ValueError(
'No hp found for model_dir {:s}'.format(model_dir))
torch.seed(hp['seed'])
self.rng = np.random.RandomState(hp['seed'])
if sigma_rec is not None:
print('Overwrite sigma_rec with {:0.3f}'.format(sigma_rec))
hp['sigma_rec'] = sigma_rec
if dt is not None:
print('Overwrite original dt with {:0.1f}'.format(dt))
hp['dt'] = dt
hp['alpha'] = 1.0 * hp['dt'] / hp['tau']
# Input, target output, and cost mask
# Shape: [Time, Batch, Num_units]
if hp['in_type'] != 'normal':
raise ValueError('Only support in_type ' + hp['in_type'])
self._build(hp)
self.model_dir = model_dir
self.hp = hp
def get_finalperformance(model_dirs):
"""Get lists of final performance."""
hp = tools.load_hp(model_dirs[0])
rule_plot = hp['rules']
final_cost, final_perf = OrderedDict(), OrderedDict()
for rule in rule_plot:
final_cost[rule] = list()
final_perf[rule] = list()
training_time_plot = list()
# Recording performance and cost for networks
for model_dir in model_dirs:
log = tools.load_log(model_dir)
if log is None:
continue
for rule in rule_plot:
final_perf[rule] += [float(log['perf_' + rule][-1])]
final_cost[rule] += [float(log['cost_' + rule][-1])]
training_time_plot.append(log['times'][-1])
return final_cost, final_perf, rule_plot, training_time_plot
def plot_posttrain_performance(posttrain_setup, trainables):
from task import rule_name
hp_target = {'posttrain_setup': posttrain_setup,
'trainables': trainables}
fs = 7
fig = plt.figure(figsize=(1.5, 1.2))
ax = fig.add_axes([0.25, 0.3, 0.7, 0.65])
colors = ['xkcd:blue', 'xkcd:red']
for pretrain_setup in [1, 0]:
c = colors[pretrain_setup]
l = ['B', 'A'][pretrain_setup]
hp_target['pretrain_setup'] = pretrain_setup
model_dirs = tools.find_all_models(DATAPATH, hp_target)
hp = tools.load_hp(model_dirs[0])
rule = hp['rule_trains'][0] # depends on posttrain setup
for model_dir in model_dirs:
log = tools.load_log(model_dir)
ax.plot(np.array(log['trials']) / 1000.,
log['perf_' + rule], color=c, alpha=0.1)
avg_perfs, trials = get_avg_performance(model_dirs, rule)
l0 = ax.plot(trials / 1000., avg_perfs, color=c, label=l)
ax.set_ylim([0, 1])
ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=2)
ax.set_yticks([0, 1])
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
# lg = ax.legend(title='Pretrained set', ncol=2, loc=4,
# frameon=False)
plt.ylabel('Perf. of ' + rule_name[rule])
# plt.title('Training ' + hp_target['trainables'])
plt.savefig('figure/Posttrain_post{:d}train{:s}.pdf'.format(
posttrain_setup, trainables), transparent=True)
def plot_hist_varprop_all(model_dir, plot_control=True):
'''
Plot histogram of proportion of variance for some tasks across units
:param save_name:
:param data_type:
:param rule_pair: list of rule_pair. Show proportion of variance for the first rule
:return:
'''
model_dirs = tools.valid_model_dirs(model_dir)
hp = tools.load_hp(model_dirs[0])
rules = hp['rules']
figsize = (7, 7)
# For testing
# rules, figsize = ['fdgo','reactgo','delaygo', 'fdanti', 'reactanti'], (4, 4)
fs = 6 # fontsize
f, axarr = plt.subplots(len(rules), len(rules), figsize=figsize)
plt.subplots_adjust(left=0.1, right=0.98, bottom=0.02, top=0.9)
for i in range(len(rules)):
for j in range(len(rules)):
ax = axarr[i, j]
if i == 0:
ax.set_title(rule_name[rules[j]],
fontsize=fs,
rotation=45,
va='bottom')
if j == 0:
ax.set_ylabel(rule_name[rules[i]],
fontsize=fs,
rotation=45,
ha='right')
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)
ax.spines["top"].set_visible(False)
if i == j:
ax.spines["bottom"].set_visible(False)
ax.set_xticks([])
ax.set_yticks([])
continue
hists, bins_edge = compute_hist_varprop(model_dir,
(rules[i], rules[j]))
hist_low, hist_med, hist_high = np.percentile(hists, [10, 50, 90],
axis=0)
hist_med /= hist_med.sum()
# Control case
if plot_control:
hists_ctrl, _ = compute_hist_varprop(model_dir,
(rules[i], rules[j]),
random_rotation=True)
_, hist_med_ctrl, _ = np.percentile(hists_ctrl, [10, 50, 90],
axis=0)
hist_med_ctrl /= hist_med_ctrl.sum()
ax.plot((bins_edge[:-1] + bins_edge[1:]) / 2,
hist_med_ctrl,
color='gray',
lw=0.75)
ax.plot((bins_edge[:-1] + bins_edge[1:]) / 2,
hist_med,
color='black')
plt.locator_params(nbins=3)
# ax.set_ylim(bottom=-0.02*hist_med.max())
ax.set_ylim([-0.01, 0.6])
print(hist_med.max())
ax.set_xticks([-1, 1])
ax.set_xticklabels([])
if i == 0 and j == 1:
ax.set_yticks([0, 0.6])
ax.spines["left"].set_visible(True)
else:
ax.set_yticks([])
ax.set_xlim([-1, 1])
ax.xaxis.set_ticks_position('bottom')
ax.tick_params(axis='both', which='major', labelsize=fs, length=2)
# plt.tight_layout()
plt.savefig('figure/plot_hist_varprop_all.pdf', transparent=True)
def __init__(self,
model_dir,
data_type,
res,
normalization_method='max'): # add res by yichen
hp = tools.load_hp(model_dir)
# If not computed, use variance.py
#fname = os.path.join(model_dir, 'variance_' + data_type + '.pkl')
#res = tools.load_pickle(fname)
h_var_all_ = res['h_var_all']
self.keys = res['keys']
# First only get active units. Total variance across tasks larger than 1e-3
# ind_active = np.where(h_var_all_.sum(axis=1) > 1e-2)[0]
ind_active = np.where(h_var_all_.sum(axis=1) > 1e-3)[0]
h_var_all = h_var_all_[ind_active, :]
# Normalize by the total variance across tasks
if normalization_method == 'sum':
h_normvar_all = (h_var_all.T / np.sum(h_var_all, axis=1)).T
elif normalization_method == 'max':
h_normvar_all = (h_var_all.T / np.max(h_var_all, axis=1)).T
elif normalization_method == 'none':
h_normvar_all = h_var_all
else:
raise NotImplementedError()
################################## Clustering ################################
from sklearn import metrics
X = h_normvar_all
# Clustering
from sklearn.cluster import AgglomerativeClustering, KMeans
# Choose number of clusters that maximize silhouette score
n_clusters = range(2, 30)
scores = list()
labels_list = list()
for n_cluster in n_clusters:
# clustering = AgglomerativeClustering(n_cluster, affinity='cosine', linkage='average')
clustering = KMeans(n_cluster,
algorithm='full',
n_init=20,
random_state=0)
clustering.fit(
X) # n_samples, n_features = n_units, n_rules/n_epochs
labels = clustering.labels_ # cluster labels
score = metrics.silhouette_score(X, labels)
scores.append(score)
labels_list.append(labels)
scores = np.array(scores)
# Heuristic elbow method
# Choose the number of cluster when Silhouette score first falls
# Choose the number of cluster when Silhouette score is maximum
if data_type == 'rule':
#i = np.where((scores[1:]-scores[:-1])<0)[0][0]
i = np.argmax(scores)
else:
# The more rigorous method doesn't work well in this case
i = n_clusters.index(10)
labels = labels_list[i]
n_cluster = n_clusters[i]
print('Choosing {:d} clusters'.format(n_cluster))
# Sort clusters by its task preference (important for consistency across nets)
if data_type == 'rule':
label_prefs = [
np.argmax(h_normvar_all[labels == l].sum(axis=0))
for l in set(labels)
]
elif data_type == 'epoch':
## TODO: this may no longer work!
label_prefs = [
self.keys[np.argmax(h_normvar_all[labels == l].sum(axis=0))][0]
for l in set(labels)
]
ind_label_sort = np.argsort(label_prefs)
label_prefs = np.array(label_prefs)[ind_label_sort]
# Relabel
labels2 = np.zeros_like(labels)
for i, ind in enumerate(ind_label_sort):
labels2[labels == ind] = i
labels = labels2
# # Sort data by labels and by input connectivity
# model = Model(save_name)
# hp = model.hp
# with tf.Session() as sess:
# model.restore(sess)
# var_list = sess.run(model.var_list)
#
# # Get connectivity
# w_out = var_list[0].T
# b_out = var_list[1]
# w_in = var_list[2][:n_input, :].T
# w_rec = var_list[2][n_input:, :].T
# b_rec = var_list[3]
#
# # nx, nh, ny = hp['shape']
# nr = hp['n_eachring']
#
# sort_by = 'w_in'
# if sort_by == 'w_in':
# w_in_mod1 = w_in[ind_active, :][:, 1:nr+1]
# w_in_mod2 = w_in[ind_active, :][:, nr+1:2*nr+1]
# w_in_modboth = w_in_mod1 + w_in_mod2
# w_prefs = np.argmax(w_in_modboth, axis=1)
# elif sort_by == 'w_out':
# w_prefs = np.argmax(w_out[1:, ind_active], axis=0)
#
# ind_sort = np.lexsort((w_prefs, labels)) # sort by labels then by prefs
###############################add by yichen############################################
#adjust the cluster order#
freq = dict()
for i in labels:
if i in freq:
freq[i] += 1
else:
freq[i] = 1
tmp_seq = sorted(
freq.items(),
key=lambda item: item[1]) #sort by frequency ,low to high
relabel_dict = dict()
for i in range(len(tmp_seq)):
relabel_dict[tmp_seq[i][0]] = i
relabel = []
for i in labels:
relabel.append(relabel_dict[i])
labels = relabel
labels = np.array(labels)
########################################################################################
ind_sort = np.argsort(labels)
labels = labels[ind_sort]
self.h_normvar_all = h_normvar_all[ind_sort, :]
self.ind_active = ind_active[ind_sort]
self.n_clusters = n_clusters
self.scores = scores
self.n_cluster = n_cluster
self.h_var_all = h_var_all
self.normalization_method = normalization_method
self.labels = labels
self.unique_labels = np.unique(labels)
self.model_dir = model_dir
self.hp = hp
self.data_type = data_type
self.rules = hp['rules']
def _plot_performanceprogress_cont(model_dir, model_dir2=None, save=True):
# Plot Training Progress
log = tools.load_log(model_dir)
hp = tools.load_hp(model_dir)
trials = np.array(log['trials']) / 1000.
times = log['times']
rule_now = log['rule_now']
if model_dir2 is not None:
log2 = tools.load_log(model_dir2)
trials2 = np.array(log2['trials']) / 1000.
fs = 7 # fontsize
lines = list()
labels = list()
rule_train_plot = hp['rule_trains']
rule_test_plot = hp['rules']
nx, ny = 4, 2
fig, axarr = plt.subplots(nx, ny, figsize=(3, 3), sharex=True)
for i in range(int(nx * ny)):
ix, iy = i % nx, int(i / nx)
ax = axarr[ix, iy]
if i >= len(rule_test_plot):
ax.axis('off')
continue
rule = rule_test_plot[i]
# Plot fills
trials_rule_prev_end = 0 # end of previous rule training time
for rule_ in rule_train_plot:
if rule == rule_:
ec = 'black'
else:
ec = (0, 0, 0, 0.1)
trials_rule_now = [trials_rule_prev_end] + [
trials[ii]
for ii in range(len(rule_now)) if rule_now[ii] == rule_
]
trials_rule_prev_end = trials_rule_now[-1]
ax.fill_between(trials_rule_now,
0,
1,
facecolor='none',
edgecolor=ec,
linewidth=0.5)
# Plot lines
line = ax.plot(trials, log['perf_' + rule], lw=1, color='gray')
if model_dir2 is not None:
ax.plot(trials2, log2['perf_' + rule], lw=1, color='red')
lines.append(line[0])
if isinstance(rule, str):
rule_name_print = rule_name[rule]
else:
rule_name_print = ' & '.join([rule_name[r] for r in rule])
labels.append(rule_name_print)
ax.tick_params(axis='both', which='major', labelsize=fs)
ax.set_ylim([0, 1.05])
ax.set_xlim([0, trials_rule_prev_end])
ax.set_yticks([0, 1])
ax.set_xticks([0, np.floor(trials_rule_prev_end / 100.) * 100])
if (ix == nx - 1) and (iy == 0):
ax.set_xlabel('Total trials (1,000)', fontsize=fs, labelpad=1)
if i == 0:
ax.set_ylabel('Performance', fontsize=fs, labelpad=1)
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('left')
ax.set_title(rule_name[rule], fontsize=fs, y=0.87, color='black')
print('Training time {:0.1f} hours'.format(times[-1] / 3600.))
if save:
name = 'TrainingCont_Progress'
if model_dir2 is not None:
name = name + '2'
plt.savefig('figure/' + name + '.pdf', transparent=True)
plt.show()
def plot_histogram():
initdict = defaultdict(list)
initdictother = defaultdict(list)
initdictotherother = defaultdict(list)
for model_dir in model_dirs:
hp = tools.load_hp(model_dir)
#check if performance exceeds target
log = tools.load_log(model_dir)
#if log['perf_avg'][-1] > hp['target_perf']:
if log['perf_min'][-1] > hp['target_perf']:
print('no. of clusters', log['n_cluster'])
n_clusters.append(log['n_cluster'])
hp_list.append(hp)
initdict[hp['w_rec_init']].append(log['n_cluster'])
initdict[hp['activation']].append(log['n_cluster'])
#initdict[hp['rnn_type']].append(log['n_cluster'])
if hp['activation'] != 'tanh':
initdict[hp['rnn_type']].append(log['n_cluster'])
initdictother[hp['rnn_type']+hp['activation']].append(log['n_cluster'])
initdictotherother[hp['rnn_type']+hp['activation']+hp['w_rec_init']].append(log['n_cluster'])
if hp['l1_h'] == 0:
initdict['l1_h_0'].append(log['n_cluster'])
else: #hp['l1_h'] == 1e-3 or 1e-4 or 1e-5:
keyvalstr = 'l1_h_1emin'+str(int(abs(np.log10(hp['l1_h']))))
initdict[keyvalstr].append(log['n_cluster'])
if hp['l1_weight'] == 0:
initdict['l1_weight_0'].append(log['n_cluster'])
else: #hp['l1_h'] == 1e-3 or 1e-4 or 1e-5:
keyvalstr = 'l1_weight_1emin'+str(int(abs(np.log10(hp['l1_weight']))))
initdict[keyvalstr].append(log['n_cluster'])
#initdict[hp['l1_weight']].append(log['n_cluster'])
# Check no of clusters under various conditions.
f, axarr = plt.subplots(7, 1, figsize=(3,12), sharex=True)
u = 0
for key in initdict.keys():
if 'l1_' not in key:
title = (key + ' ' + str(len(initdict[key])) +
' mean: '+str(round(np.mean(initdict[key]),2)))
axarr[u].set_title(title)
axarr[u].hist(initdict[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)
# plt.savefig('./figure/histforcases_96nets.png')
# plt.savefig('./figure/histforcases__pt9_192nets.pdf')
# plt.savefig('./figure/histforcases___leakygrunotanh_pt9_192nets.pdf')
f, axarr = plt.subplots(4, 1, figsize=(3,8), sharex=True)
u = 0
for key in initdictother.keys():
if 'l1_' not in key:
axarr[u].set_title(key + ' ' + str(len(initdictother[key]))+ ' mean: '+str(round(np.mean(initdictother[key]),2)) )
axarr[u].hist(initdictother[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)
# plt.savefig('./figure/histforcases__leakyrnngrurelusoftplus_pt9_192nets.pdf')
f, axarr = plt.subplots(4, 1, figsize=(3,6), sharex=True)
u = 0
for key in initdictotherother.keys():
if 'l1_' not in key and 'diag' not in key:
axarr[u].set_title(key + ' ' + str(len(initdictotherother[key]))+ ' mean: '+str(round(np.mean(initdictotherother[key]),2)) )
axarr[u].hist(initdictotherother[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)
# plt.savefig('./figure/histforcases_randortho_notanh_pt9_192nets.pdf')
f, axarr = plt.subplots(4, 1, figsize=(3,6),sharex=True)
u = 0
for key in initdictotherother.keys():
if 'l1_' not in key and 'randortho' not in key:
axarr[u].set_title(key + ' ' + str(len(initdictotherother[key]))+ ' mean: '+str(round(np.mean(initdictotherother[key]),2)) )
axarr[u].hist(initdictotherother[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)
# plt.savefig('./figure/histforcases_diag_notanh_pt9_192nets.pdf')
#regu--
f, axarr = plt.subplots(4, 1,figsize=(3,8),sharex=True)
u = 0
for key in initdict.keys():
if 'l1_h_' in key:
axarr[u].set_title(key + ' ' + str(len(initdict[key]))+ ' mean: '+str(round(np.mean(initdict[key]),2)) )
axarr[u].hist(initdict[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)
#plt.savefig('./figure/noofclusters_pt9_l1_h_192nets.pdf')
f, axarr = plt.subplots(4, 1,figsize=(3,8),sharex=True)
u = 0
for key in initdict.keys():
if 'l1_weight_' in key:
axarr[u].set_title(key + ' ' + str(len(initdict[key])) + ' mean: '+str(round(np.mean(initdict[key]),2)) )
axarr[u].hist(initdict[key])
u += 1
f.subplots_adjust(wspace=.3, hspace=0.3)