replace=False,
size=200)
if set(sample) in samples:
continue
samples.append(set(sample))
samp_bert_embeds = bert_embeds[sample]
samp_subj = subj[sample]
samp_obj = obj[sample]
samp_verb = verb[sample]
samp_adj1 = adj1[sample]
samp_adj2 = adj2[sample]
samp_rand_verb = rand_verb[sample]
bert_geom = RSA.calculate_geometry(samp_bert_embeds)
subj_geom = RSA.calculate_geometry(samp_subj)
obj_geom = RSA.calculate_geometry(samp_obj)
verb_geom = RSA.calculate_geometry(samp_verb)
rand_verb_geom = RSA.calculate_geometry(samp_rand_verb)
adj1_geom = RSA.calculate_geometry(samp_adj1)
adj2_geom = RSA.calculate_geometry(samp_adj2)
rsa_subj_dist.append(spearmanr([subj_geom, bert_geom], axis=1)[0])
rsa_obj_dist.append(spearmanr([obj_geom, bert_geom], axis=1)[0])
rsa_verb_dist.append(spearmanr([verb_geom, bert_geom], axis=1)[0])
rsa_adj1_dist.append(spearmanr([adj1_geom, bert_geom], axis=1)[0])
rsa_adj2_dist.append(spearmanr([adj2_geom, bert_geom], axis=1)[0])
rsa_rand_verb_dist.append(
spearmanr([rand_verb_geom, bert_geom], axis=1)[0])
# Generate 100 samples of representational similarity
while len(samples) < 100:
sample = np.random.choice(range(0, len(glove_list)),
replace=False,
size=200)
if set(sample) in samples:
continue
samples.append(set(sample))
samp_bert_embeds = bert_embeds[sample]
samp_glove_ant = glove_ant[sample]
samp_glove_nonant = glove_nonant[sample]
samp_glove_rand = glove_rand[sample]
bert_geom = RSA.calculate_geometry(samp_bert_embeds)
ant_geom = RSA.calculate_geometry(samp_glove_ant)
nonant_geom = RSA.calculate_geometry(samp_glove_nonant)
rand_geom = RSA.calculate_geometry(samp_glove_rand)
rsa_ant_dist.append(spearmanr([ant_geom, bert_geom], axis=1)[0])
rsa_nonant_dist.append(spearmanr([nonant_geom, bert_geom], axis=1)[0])
rsa_rand_dist.append(spearmanr([rand_geom, bert_geom], axis=1)[0])
# Run tests and generate plots
print(
f'RSA Verb + Antecedent: {np.mean(rsa_ant_dist)} STD: {np.std(rsa_ant_dist)}'
)
print(
f'RSA Verb + Non-Antecedent: {np.mean(rsa_nonant_dist)} STD: {np.std(rsa_nonant_dist)}'
)
# Generate 100 samples of representational similarity
while len(samples) < 100:
sample = np.random.choice(range(0, len(glove_list)),
replace=False,
size=200)
if set(sample) in samples:
continue
samples.append(set(sample))
samp_bert_embeds = bert_embeds[sample]
samp_glove_subj = glove_subj[sample]
samp_glove_nonarg = glove_nonarg[sample]
samp_glove_rand = glove_rand[sample]
bert_geom = RSA.calculate_geometry(samp_bert_embeds)
subj_geom = RSA.calculate_geometry(samp_glove_subj)
nonarg_geom = RSA.calculate_geometry(samp_glove_nonarg)
rand_geom = RSA.calculate_geometry(samp_glove_rand)
rsa_subj_dist.append(spearmanr([subj_geom, bert_geom], axis=1)[0])
rsa_nonarg_dist.append(spearmanr([nonarg_geom, bert_geom], axis=1)[0])
rsa_rand_dist.append(spearmanr([rand_geom, bert_geom], axis=1)[0])
# Perform tests and generate plots
print(
f'RSA Verb + Subject: {np.mean(rsa_subj_dist)} STD: {np.std(rsa_subj_dist)}'
)
print(
f'RSA Verb + Non-Argument: {np.mean(rsa_nonarg_dist)} STD: {np.std(rsa_nonarg_dist)}'
)
while len(samples) < 100:
sample = np.random.choice(range(0, len(glove_list)),
replace=False,
size=50)
if set(sample) in samples:
continue
samples.append(set(sample))
samp_bert_embeds = bert_embeds[sample]
samp_subj = subj[sample]
samp_verb = verb[sample]
samp_rand_verb = rand_verb[sample]
bert_geom = RSA.calculate_geometry(samp_bert_embeds)
subj_geom = RSA.calculate_geometry(samp_subj)
verb_geom = RSA.calculate_geometry(samp_verb)
rand_verb_geom = RSA.calculate_geometry(samp_rand_verb)
rsa_subj_dist.append(spearmanr([subj_geom, bert_geom], axis=1)[0])
rsa_verb_dist.append(spearmanr([verb_geom, bert_geom], axis=1)[0])
rsa_rand_verb_dist.append(
spearmanr([rand_verb_geom, bert_geom], axis=1)[0])
# Run Tests
print(
f'RSA Subject: {np.mean(rsa_subj_dist)} STD: {np.std(rsa_subj_dist)}')
print(f'RSA Verb: {np.mean(rsa_verb_dist)} STD: {np.std(rsa_verb_dist)}')
print(
f'RSA Random Verb: {np.mean(rsa_rand_verb_dist)} STD: {np.std(rsa_rand_verb_dist)}'
