def information(self, test_var, normalize=False):
"""
Computes the mutual information between the output of the task, and another variable
"""
# given b is positive
ab = util.decimal(self(test_var)).numpy()
# pab = np.unique(ab, return_counts=True)[1]/np.min([self.num_cond,(2**self.num_var)])
pab = np.unique(ab, return_counts=True)[1] / len(test_var)
# given b is negative
b_ = np.setdiff1d(range(self.num_cond), test_var)
ab_ = util.decimal(self(b_)).numpy()
# pab_ = np.unique(ab_, return_counts=True)[1]/np.min([self.num_cond,(2**self.num_var)])
pab_ = np.unique(ab_, return_counts=True)[1] / len(test_var)
# entropy of outputs (in case they are degenerate)
a = util.decimal(self(np.arange(self.num_cond)))
pa = np.unique(a, return_counts=True)[1] / self.num_cond
Ha = -np.sum(pa * np.log2(pa))
# I(a,b) = H(a) - H(a | b)
MI = Ha + 0.5 * (np.sum(pab * np.log2(pab)) +
np.sum(pab_ * np.log2(pab_)))
return MI
def __call__(self, labels, noise=None):
these = np.array([np.isin(labels, p)
for p in self.positives]).astype(float)
if self.use_mse:
return assistants.Indicator(self.dim_output, self.dim_output)(
util.decimal(these.T).astype(int)).float()
else:
return torch.tensor(util.decimal(these.T)).int()
if which_data == 'assoc':
p = 2**num_var
allowed_actions = [0, 1, 2]
# allowed_actions = [0,1,2,4]
# allowed_actions = [0]
p_action = [0.7, 0.15, 0.15]
# p_action = [0.61, 0.13, 0.13, 0.13]
# p_action = [1.0]
output_states = (this_exp.train_data[0][:ndat, :].data + 1) / 2
# output_states = this_exp.train_data[1][:ndat,:].data
input_states = (this_exp.train_data[0][:ndat, :].data + 1) / 2
abstract_conds = util.decimal(this_exp.train_data[1])[:ndat]
cond_set = np.unique(abstract_conds)
# draw the "actions" for each data point
actns = torch.tensor(np.random.choice(allowed_actions, ndat,
p=p_action)).int()
actions = torch.stack([(actns & (2**i)) / 2**i
for i in range(num_var)]).float().T
# act_rep = assistants.Indicator(p,p)(util.decimal(actions).int())
act_rep = actions.data
# inputs = np.concatenate([input_states,act_rep], axis=1)
# # inputs = np.concatenate([input_states, this_exp.train_data[1]], axis=1)
inputs = input_states.float()
#%% Projecting onto specific dichotmy subspaces these_dichotomies = (0,9) this_network = -1 model = all_nets[0][this_network] args = all_args[0][this_network] vecs = coding_vectors[this_network] this_exp.load_other_info(args) this_exp.load_data(SAVE_DIR) z = model(this_exp.train_data[0])[2].detach().numpy() fake_task = util.RandomDichotomies(d=this_exp.task.positives) colorby = util.decimal(fake_task(this_exp.train_conditions)[:5000]) # vals = util.cosine_sim() orth_vecs = la.orth(vecs[these_dichotomies,...].squeeze().T) # basis for subspace Proj = orth_vecs@orth_vecs.T z_proj = [email protected] U, S, _ = la.svd(z_proj-z_proj.mean(1)[:,None], full_matrices=False) pcs = z_proj.T[:5000,:]@U[:3,:].T proj_perf = spc.expit(model.dec(torch.tensor(z_proj).float().T).detach())[fake_task(this_exp.train_conditions)==1].mean() # plt.figure()
# apply_rotation = True
input_task = util.RandomDichotomies(d=[(0, 1, 2, 3), (0, 2, 4, 6), (0, 1, 4,
5)])
# output_task = util.RandomDichotomies(d=[(0,1,6,7), (0,2,5,7)]) # xor of first two
output_task = util.RandomDichotomies(d=[(0, 3, 5, 6)]) # 3d xor
# output_task = util.RandomDichotomies(d=[(0,1,6,7), (0,4,5,6)]) # 3d xor
# output_task = util.RandomDichotomies(d=[(0,1,2,3),(0,2,4,6)])
# output_task = util.RandomDichotomies(d=[(0,1,4,5),(0,2,5,7),(0,1,6,7)]) # 3 incompatible dichotomies
# input_task = util.RandomDichotomies(d=[(0,1),(0,2)])
# output_task = util.RandomDichotomies(d=[(0,1)])
inp_condition = np.random.choice(2**num_var, num_data)
# var_bit = (np.random.rand(num_var, num_data)>0.5).astype(int)
var_bit = input_task(inp_condition).numpy().T
inp_subcondition = util.decimal(var_bit[:num_recoded, :].T).astype(int)
means = np.random.randn(num_var, dim_inp)
cat_means = np.random.randn(2**num_recoded, dim_inp * num_recoded)
mns = (means[:, None, :] * var_bit[:, :, None]) - (means[:, None, :] *
(1 - var_bit[:, :, None]))
clus_mns = np.reshape(mns.transpose((0, 2, 1)), (dim_inp * num_var, -1)).T
clus_mns[:, :num_recoded * dim_inp] = cat_means[inp_subcondition, :]
if apply_rotation:
C = np.random.rand(num_var * dim_inp, num_var * dim_inp)
clus_mns = clus_mns @ la.qr(C)[0][:num_var * dim_inp, :]
inputs = torch.tensor(clus_mns + np.random.randn(num_data, num_var * dim_inp) *
p_action = [0.8,0.1,0.1] # p_action = [1.0] # output_states = this_exp.train_data[1].data # output_states = util.decimal(this_exp.train_data[1]) output_states = this_exp.train_data[0].data # output_states = ContinuousEmbedding(N_, 1.0)(this_exp.train_data[1]) # output_states = assistants.Indicator(p,p)(util.decimal(this_exp.train_data[1]).int()) input_states = this_exp.train_data[0].data # input_states = 1*this_exp.train_data[1].data # input_states = assistants.Indicator(p,p)(util.decimal(this_exp.train_data[1]).int())@W2.T+np.random.randn(56000,N_)*0.2 # input_states = this_exp.train_data[1]@W1.T + np.random.randn(56000,N_)*0.2 abstract_conds = util.decimal(this_exp.train_data[1]) cond_set = np.unique(abstract_conds) # draw the "actions" for each data point actns = torch.tensor(np.random.choice(allowed_actions, this_exp.train_data[0].shape[0], p=p_action)).int() actions = torch.stack([(actns&(2**i))/2**i for i in range(num_var)]).float().T # act_rep = assistants.Indicator(p,p)(util.decimal(actions).int()) act_rep = actions.data # inputs = np.concatenate([input_states,act_rep], axis=1) # # inputs = np.concatenate([input_states, this_exp.train_data[1]], axis=1) inputs = input_states.float().detach().numpy() # # sample the successor states, i.e. input + action successors = np.mod(this_exp.train_data[1]+actions, 2)
# fake_task.positives = [(0,1,2,6)]
idx = np.random.choice(inputs.shape[0], n_compute, replace=False)
z = model(inputs[idx,...].float()).detach().numpy()
# ans = this_exp.train_data[1][idx,...]
# ans = output_task(this_exp.train_conditions)[idx]
# cond = util.decimal(ans)
# cond = this_exp.train_conditions[idx]
cond = inp_condition[idx]
# colorby = cond
# colorby = this_exp.train_conditions[idx]
point_col = util.decimal(fake_task(cond))
centr_col = util.decimal(fake_task(np.unique(cond)))
mds = manifold.MDS(n_components=n_mds)
# emb = mds.fit_transform(la.norm(z.T[:,:,None] - z.T[:,None,:],axis=0))
# emb = mds.fit(z)
emb = mds.fit_transform(np.round(z,2))
if n_mds == 2:
scat = plt.scatter(emb[:,0],emb[:,1], c=colorby)
plt.xlabel('MDS1')
plt.ylabel('MDS2')
elif n_mds == 3:
# print('Epoch %d: loss=%.3f'%(epoch, running_loss/(i+1)))
#%%
n_mds = 2
n_compute = 500
idx = np.random.choice(this_exp.train_data[0].shape[0],
n_compute,
replace=False)
z = (W @ (this_exp.train_data[0][idx, ...].detach().numpy().T) + b).T
# z = lin(this_exp.train_data[0])[idx,...].detach().numpy()
# z = targ[idx,...]
# ans = this_exp.train_conditions[idx,...]
ans = this_exp.train_data[1][idx, ...]
cond = util.decimal(ans)
mds = manifold.MDS(n_components=2)
emb = mds.fit_transform(z)
scat = plt.scatter(emb[:, 0], emb[:, 1], c=cond)
plt.xlabel('MDS1')
plt.ylabel('MDS2')
cb = plt.colorbar(scat,
ticks=np.unique(cond),
drawedges=True,
values=np.unique(cond))
cb.set_ticklabels(np.unique(cond) + 1)
cb.set_alpha(1)
cb.draw_all()
ax=axs[0])
an2 = dicplt.LineAnime(weights_proj[:, :, 0].T,
weights_proj[:, :, 1].T,
colors=cm.bwr(grp / 1),
ax=axs[1])
dicplt.AnimeCollection(an1, an2).save(SAVE_DIR + '/vidya/tempmovie.mp4',
fps=30)
#%%
this_grp = 0
ax = dicplt.scatter3d(8 * inp_coefs.T,
s=200,
marker='*',
c=cm.viridis(util.decimal(y_.T) / 7))
dicplt.LineAnime3D(
weights_proj[:, grp == this_grp, 0].T,
weights_proj[:, grp == this_grp, 1].T,
weights_proj[:, grp == this_grp, 2].T,
view_period=100,
ax=ax,
colors=cm.viridis(grp[grp == this_grp] / 7),
rotation_period=500).save(SAVE_DIR + 'vidya/tempmovie_%s.mp4' %
grp_weights[:, this_grp])
#%%
p_abs = 0.0
inp_align = []
dim_inp = 32 # dimension per variable
num_data = 5000 # total
noise = 0.05
switch_fraction = 0.1
input_task = util.StandardBinary(3)
# output_task = util.RandomDichotomies(d=[(0,1,6,7), (0,2,5,7)]) # xor of first two
output_task = util.RandomDichotomies(d=[(0, 3, 5, 6)]) # 3d xor
inp_condition = np.random.choice(2**3, num_data)
var_bit = input_task(inp_condition).numpy().T
action_outcome = var_bit[:2, :]
context = var_bit[2, :]
stimulus = util.decimal(action_outcome.T).astype(int)
stimulus[context == 1] = np.mod(stimulus[context == 1] + 1,
4) # effect of context
means_pos = np.random.randn(2, dim_inp)
means_neg = np.random.randn(2, dim_inp)
stim_pattern = np.random.randn(4, dim_inp)
mns = (means_pos[:, None, :] *
action_outcome[:, :, None]) + (means_neg[:, None, :] *
(1 - action_outcome[:, :, None]))
ao_t = np.reshape(mns.transpose((0, 2, 1)), (dim_inp * 2, -1)).T
s_t = stim_pattern[stimulus, :]
prev_trial = np.arange(num_data)
#%%
num_cond = 2**10
num_var = 10
task = util.RandomDichotomies(num_cond,num_var,0)
# task = util.ParityMagnitude()t
# this_exp = exp.mnist_multiclass(task, SAVE_DIR, abstracts=abstract_variables)
this_exp = exp.random_patterns(task, SAVE_DIR,
num_class=num_cond,
dim=100,
var_means=1)
#%% Rotation from square to tetrahedron
abstract_conds = util.decimal(this_exp.train_data[1])
dim = 50
noise = 0.2
ndat = 2000
# lin_clf = svm.LinearSVC
# lin_clf = linear_model.LogisticRegression
# lin_clf = linear_model.Perceptron
# lin_clf = linear_model.RidgeClassifier
emb = util.ContinuousEmbedding(dim, 0.0)
z = emb(this_exp.train_data[1])
_, _, V = la.svd(z-z.mean(0)[None,:],full_matrices=False)
# V = V[:3,:]
cond = this_exp.train_conditions
