def get_adj_matrix(filename,
old_label=None,
new_label=None,
new_phrase=None,
old_phrase=None,
new_emb_dict=None,
new_phrase_emb_dict=None):
"""Get the adjacency matrix."""
# Load scene graph objects and phrase embeddings
npzfile = np.load(filename, allow_pickle=True)
sg_objects = npzfile['name1'].tolist()
phrase_emb = npzfile['name2']
disc_indices_array = npzfile['name3'].tolist()
match_array = npzfile['name4']
# Change the old label to new label
if old_label and new_label:
sg_objects = change_label(sg_objects, old_label, new_label,
new_emb_dict)
# If a new phrase is used for comparing
if new_phrase:
# If new phrase emb dict is used
if new_phrase_emb_dict:
# Get new phrase embedding (saved merged embeddings) -- not used
phrase_emb = new_phrase_emb_dict[old_phrase]
# Get combination of old and new phrase
combo_emb_array = np.zeros((2, 300))
combo_emb_array[0, :] = emb.embed(old_phrase)
combo_emb_array[1, :] = emb.embed(new_phrase)
combo_emb_array = combo_emb_array.mean(axis=0)
phrase_emb = combo_emb_array
# Otherwise just embed new phrase
else:
phrase_emb = emb.embed(new_phrase)
# Get all embeddings
embeddings = np.array(sg_objects[1])
# Get cosine similarity
cosim = 1 - cosine_similarity(phrase_emb.reshape(1, -1), embeddings)[0]
# Convert everything close 0 to 0.0
cosim[np.isclose(0.0, cosim)] = 0.0
# Change cosine simiarlity to 1.0 if object is to be discarded
for ind in disc_indices_array:
cosim[ind] = 1.0
return cosim, match_array
def update_embeddings(self, params, data=[]):
for j in range(params['num_agents']):
if params['embedding'] == 'a_s':
self.embeddings[j] = [
embed(params, [], self.agents[j], self.selected)
]
else:
self.embeddings[j] = [
embed(params, s, self.agents[j], self.selected)
for s in data[j][1]
]
def change_label(original_sg_objects, old_label, new_label, new_emb_dict=None):
"""Change old label to new label."""
# Initialize
new_labels = []
new_embeddings = []
sg_objects = [[], [], []]
# For each label in the original objects
for i, label in enumerate(original_sg_objects[0]):
# Get category and ID
cat, ID = label.split('-')
# If cat is the old label that is to be replaced
if cat == old_label:
# Save new label
new_labels.append(new_label + '-' + ID)
# If new embeddings dict, combine old and new label embedding
if new_emb_dict:
# Get new object embedding (saved merged embeddings) -- not used
obj_emb = new_emb_dict[old_label]
# Get combined embedding for old and new label
combo_emb_array = np.zeros((2, 300))
combo_emb_array[0, :] = emb.embed(old_label)
combo_emb_array[1, :] = emb.embed(new_label)
combo_emb_array = combo_emb_array.mean(axis=0)
obj_emb = combo_emb_array
# Ohterwise only new embedding
else:
obj_emb = emb.embed(new_label)
# Save new embedding
new_embeddings.append(obj_emb)
# If not label to be changed, save original data
else:
new_labels.append(label)
new_embeddings.append(original_sg_objects[1][i])
# Get new labels, embeddings and coordinates
sg_objects[0] = new_labels
sg_objects[1] = new_embeddings
sg_objects[2] = original_sg_objects[2]
return sg_objects
def train(params):
env = gym.make(params['env_name'])
params['ob_dim'] = env.observation_space.shape[0]
params['ac_dim'] = env.action_space.shape[0]
master = Learner(params)
n_eps = 0
n_iter = 0
ts_cumulative = 0
ts, rollouts, rewards, max_rwds, dists, min_dists, agents, lambdas = [0], [0], [], [], [], [], [], []
params['num_sensings'] = params['sensings']
master.agent=0
# get initial states so you can get behavioral embeddings
population = [master.agents[x].rollout(env, params['steps'], incl_data=True) for x in master.agents.keys()]
all_states = [s[0] for x in population for s in x[1]]
master.selected = select_states(master, params, all_states)
master.update_embeddings(params, population)
master.calc_pairwise_dists(params)
master.select_agent()
master.get_agent()
#initial reward
reward = master.policy.rollout(env, params['steps'], incl_data=False)
rewards.append(reward)
agents.append(master.agent)
dists.append(master.dists)
max_reward=reward
max_rwds.append(max_reward)
min_dists.append(master.min_dist)
if params['w_nov'] < 0:
bb = BayesianBandits()
params['w_nov'] = 0
lambdas.append(params['w_nov'])
while n_iter < params['max_iter']:
print('Iter: %s, Eps: %s, Mean: %s, Max: %s, Best: %s, MeanD: %s, MinD: %s, Lam: %s' %(n_iter, n_eps, np.round(reward,4), np.round(max_reward,4), master.agent, np.round(master.dists,4), np.round(master.min_dist,4), params['w_nov']))
if (n_iter>0) & (params['num_agents'] > 1):
master.calc_pairwise_dists(params)
master.select_agent()
master.get_agent()
## Main Function Call
params['n_iter'] = n_iter
if params['num_agents'] > 1:
gradient, timesteps = population_update(master, params)
n_eps += 2*params['num_sensings'] * params['num_agents']
else:
gradient, timesteps = individual_update(master, params)
n_eps += 2*params['num_sensings']
ts_cumulative += timesteps
all_states += master.states
if params['num_sensings'] < len(all_states):
all_states = sample(all_states, params['num_sensings'])
gradient /= (np.linalg.norm(gradient) / master.policy.N + 1e-8)
n_iter += 1
update = Adam(gradient, master, params['learning_rate'], n_iter)
rwds, trajectories = [], []
if params['num_evals'] > 0:
seeds = [int(np.random.uniform()*10000) for _ in range(params['num_evals'])]
for i in range(params['num_agents']):
master.agent = i
master.get_agent()
master.policy.update(master.policy.params + update[(i*master.policy.N):((i+1)*master.policy.N)])
if params['num_evals'] > 0:
reward = 0
for j in range(params['num_evals']):
r, traj = master.policy.rollout(env, params['steps'], incl_data=True, seed =seeds[j])
reward += r
reward /= params['num_evals']
else:
reward, traj = master.policy.rollout(env, params['steps'], incl_data=True)
rwds.append(reward)
trajectories.append(traj)
if reward > master.best[i]:
master.best[i] = reward
np.save('data/%s/weights/Seed%s_Agent%s' %(params['dir'], params['seed'], i), master.policy.params)
master.reward[i].append(reward)
master.update_agent()
reward = np.mean(rwds)
max_reward = max(rwds)
traj = trajectories[np.argmax(rwds)]
master.agent = np.argmax(rwds)
# Update selected states
master.selected = select_states(master, params, all_states)
master.update_embeddings(params)
master.embedding = embed(params, traj, master.policy, master.selected)
rewards.append(reward)
max_rwds.append(max_reward)
master.reward[master.agent].append(reward)
if reward > master.best[master.agent]:
master.best[master.agent] = reward
np.save('data/%s/weights/Seed%s_Agent%s' %(params['dir'], params['seed'], master.agent), master.policy.params)
## update the bandits
try:
bb.update_dists(reward)
params['w_nov'] = bb.sample()
except NameError:
pass
lambdas.append(params['w_nov'])
rollouts.append(n_eps)
agents.append(master.agent)
dists.append(master.dists)
min_dists.append(master.min_dist)
ts.append(ts_cumulative)
master.update_agent()
if n_iter % params['flush'] == 0:
reset_ray(master, params)
master.init_workers(params)
out = pd.DataFrame({'Rollouts': rollouts, 'Reward': rewards, 'Max': max_rwds, 'Timesteps': ts, 'Dists': dists, 'Min_Dist':min_dists, 'Agent': agents, 'Lambda': lambdas})
out.to_csv('data/%s/results/Seed%s.csv' %(params['dir'], params['seed']), index=False)
def word2vec(self, word):
# XXX
return embed(word)
def get_phrase_alternatives(image_ID, caption_ID, phrase_indices, sg_object_array,\
res_matrix, default_adj_matrix, default_score,\
default_alignment_acc, ONLY_GOOD_RELATIONS, ent_IDs,\
emb_version=False):
"""Get alternative phrases and evaluate on phrase grounding task."""
# Initialize
better_scores = 0
total_scores = 0
successful_relations = []
successful_relations_align = []
total_relations = []
phrase_alignments_per_rel = []
alignment_accuracies = []
better_alignments = 0
same_alignments = 0
total_alignments = 0
combined_better_alignment = 0
combined_same_alignment = 0
# Load altenrative phrases
with open(
"../ConceptNet/data/alternatives_red/alternatives_phrase_" +
image_ID + "_" + caption_ID + ".pickle", "rb") as input_file:
alternative_phrases = pickle.load(input_file)
# Load specific alternatives (based on relation)
if ONLY_GOOD_RELATIONS:
# Load relations and sucess rates
with open('conceptnet_data/useful_phrase_relations.pickle',
'rb') as handle:
useful_phrase_relations = pickle.load(handle)
with open('conceptnet_data/phrase_relations_acc_dict.pickle',
'rb') as handle:
phrase_relations_acc_dict = pickle.load(handle)
# Get useful phrase relations (minimal success rate and occurences)
useful_phrase_relations = [
rel for rel in phrase_relations_acc_dict.keys()
if phrase_relations_acc_dict[rel][0] >= 0.75
and phrase_relations_acc_dict[rel][1] > 25
]
# Get alternative phrases based on acceptable relations
alternative_phrases = [(_, old_label, new_label, relation, _,
path_length)
for (_, old_label, new_label, relation, _,
path_length) in alternative_phrases
if tuple(relation) in useful_phrase_relations]
# Initialize
new_phrase_dict = {}
new_combo_phrase_emb = {}
old_phrase_index = {}
# Get dictionary for new phrases
for _, old_label, _, _, _, _ in alternative_phrases:
new_phrase_dict[old_label] = []
for _, old_label, new_label, _, _, _ in alternative_phrases:
new_phrase_dict[old_label].append(new_label)
# Get combo embeddings array for phrases
for ol_idx, old_label in enumerate(new_phrase_dict.keys()):
# Get new phrases
new_labels = new_phrase_dict[old_label]
old_phrase_index[old_label] = ol_idx
# Create array for combined embeddings for phrases
new_shape = [len(new_labels) + 1] + list(
emb.embed(new_labels[0]).shape)
combo_emb_array = np.zeros(tuple(new_shape))
# Use original phrase as first embedding
combo_emb_array[0, :] = emb.embed(old_label)
# Add embeddings for new phrases
for label_idx, new_label in enumerate(new_labels):
combo_emb_array[label_idx + 1, :] = emb.embed(new_label)
# Get mean over embeddings and save
combo_emb_array = combo_emb_array.mean(axis=0)
new_combo_phrase_emb[old_label] = combo_emb_array
# Create combo matrix and fill in original
combo_matrix = np.zeros((len(alternative_phrases) + 1, len(phrase_indices),
len(sg_object_array[0])))
combo_matrix[0, :, :] = default_adj_matrix
processed_ones = []
# Iterate over phrases
for alt_phrase_id, alt_phrase in enumerate(alternative_phrases):
# Create array
adj_matrix = np.zeros((len(phrase_indices), len(sg_object_array[0])))
# Get new phrases
new_phrase_ID, old_phrase, new_phrase, relation, _, path_length = alt_phrase
# Discard disqualified relations
if ONLY_GOOD_RELATIONS:
if tuple(relation) not in useful_phrase_relations:
continue
# Iterate over phrase set
for i, phrase_id in enumerate(phrase_indices):
# Generate filename
filename = './.cosim_collection/' + image_ID + '_' + caption_ID + '_' + phrase_id + '.npz'
# If phrase is phrase to be replaced
if new_phrase_ID == phrase_id:
# Get cosine similarity (either with combined embeddings or single)
if emb_version:
cosim, _ = get_adj_matrix(
filename,
new_phrase=new_phrase,
old_phrase=old_phrase,
new_phrase_emb_dict=new_combo_phrase_emb)
else:
cosim, _ = get_adj_matrix(filename, new_phrase=new_phrase)
# Get cosine similarity without change
else:
cosim, _ = get_adj_matrix(filename)
# Save cosine similarity
adj_matrix[i, :] = cosim
# Save cosine similarities as adj matrix for all phrases
combo_matrix[old_phrase_index[old_phrase] + 1, :, :] = adj_matrix
# Get weighted sum score
score = np.sum(np.multiply(adj_matrix, res_matrix))
# If score is smaller than default score (better!), consider success
# Optional: set minimum difference
if score < default_score: # and ((score-default_score) >=0.1):
better_scores += 1
successful_relations.append(tuple(relation))
# Track and count relations
total_relations.append(tuple(relation))
total_scores += 1
# Get phrase grounding accuracy
alignment_acc = get_alignment(adj_matrix, phrase_indices,
sg_object_array, ent_IDs)
# Check whether better/worse/same alignment
if alignment_acc > default_alignment_acc:
better_alignments += 1
if alignment_acc == default_alignment_acc:
same_alignments += 1
total_alignments += 1
# Determine whether replacement is successful
if alignment_acc >= default_alignment_acc:
successful_relations_align.append(tuple(relation))
# Save relations and accuracies
phrase_alignment_per_rel = (tuple(relation), alignment_acc)
alignment_accuracies.append(alignment_acc)
# Get mean of cosine similarities
adj_matrix = combo_matrix.mean(axis=0)
# Get accuracy for combined version
combo_alignment_acc = get_alignment(adj_matrix, phrase_indices,
sg_object_array, ent_IDs)
# Count improvements/non-improvements
if combo_alignment_acc > default_alignment_acc:
combined_better_alignment += 1
elif combo_alignment_acc == default_alignment_acc:
combined_same_alignment += 1
return better_scores, total_scores, successful_relations, total_relations, \
phrase_alignments_per_rel, alignment_accuracies, better_alignments, same_alignments, total_alignments, \
combo_alignment_acc, combined_better_alignment, combined_same_alignment, successful_relations_align
def get_sg_alternatives(image_ID, caption_ID, phrase_indices,
sg_object_array, res_matrix, default_adj_matrix,\
default_score, default_alignment_acc, ONLY_GOOD_RELATIONS, \
ent_IDs, emb_version=False):
"""Get scene graph object alternatives and evaluate on phrase grounding task."""
# Initialize
better_scores = 0
total_scores = 0
successful_relations = []
successful_relations_align = []
total_relations = []
sg_alignments_per_rel = []
alignment_accuracies = []
better_alignments = 0
same_alignments = 0
total_alignments = 0
combined_better_alignment = 0
combined_same_alignment = 0
# Load alternatives and relation accuracies (success rate)
with open(
"../ConceptNet/data/alternatives_red/alternatives_sg_" + image_ID +
"_" + caption_ID + ".pickle", "rb") as input_file:
alternative_sg_objs = pickle.load(input_file)
if ONLY_GOOD_RELATIONS:
with open('conceptnet_data/useful_sg_relations.pickle',
'rb') as handle:
useful_sg_relations = pickle.load(handle)
with open('conceptnet_data/sg_relations_acc_dict.pickle',
'rb') as handle:
sg_relations_acc_dict = pickle.load(handle)
# Filter for alternatives with relation accuracy (sucess rate) greather than 0.75 and more than 25 instances
useful_sg_relations = [
rel for rel in sg_relations_acc_dict.keys()
if sg_relations_acc_dict[rel][0] >= 0.75
and sg_relations_acc_dict[rel][1] > 25
]
alternative_sg_objs = [(_, old_label, new_label, relation, _,
path_length)
for (_, old_label, new_label, relation, _,
path_length) in alternative_sg_objs
if tuple(relation) in useful_sg_relations]
# Initalize
new_label_dict = {}
new_combo_label_emb = {}
old_label_index = {}
# Create new label dictionary
for _, old_label, _, _, _, _ in alternative_sg_objs:
new_label_dict[old_label] = []
for _, old_label, new_label, _, _, _ in alternative_sg_objs:
new_label_dict[old_label].append(new_label)
# Get combination embedding for new labels
for ol_idx, old_label in enumerate(new_label_dict.keys()):
# Get new labels
old_label_index[old_label] = ol_idx
new_labels = new_label_dict[old_label]
# Create array
new_shape = [len(new_labels) + 1] + list(
emb.embed(new_labels[0]).shape)
combo_emb_array = np.zeros(tuple(new_shape))
# Use old label as first embedding
combo_emb_array[0, :] = emb.embed(old_label)
# Add new labels to embedding array
for label_idx, new_label in enumerate(new_labels):
combo_emb_array[label_idx + 1, :] = emb.embed(new_label)
# Get mean and save combo embedding
combo_emb_array = combo_emb_array.mean(axis=0)
new_combo_label_emb[old_label] = combo_emb_array
# Create combo matrix array
combo_matrix = np.zeros((len(alternative_sg_objs) + 1, len(phrase_indices),
len(sg_object_array[0])))
# Fill with default
combo_matrix[0, :, :] = default_adj_matrix
processed_ones = []
# Iterate over every alternative object (label)
for alt_sg_id, alt_sg in enumerate(alternative_sg_objs):
# Get elements
_, old_label, new_label, relation, _, path_length = alt_sg
# Create adj matrix array
adj_matrix = np.zeros((len(phrase_indices), len(sg_object_array[0])))
# Discard relations that are not used
if ONLY_GOOD_RELATIONS:
if tuple(relation) not in useful_sg_relations:
continue
# Iterate over each phrase
for i, phrase_id in enumerate(phrase_indices):
# Get filename for loading data
filename = './.cosim_collection/' + image_ID + '_' + caption_ID + '_' + phrase_id + '.npz'
# Get cosine similarity
if emb_version:
cosim, _ = get_adj_matrix(filename,
old_label,
new_label,
new_emb_dict=new_combo_label_emb)
else:
cosim, _ = get_adj_matrix(filename, old_label, new_label)
# Save cosine similarity for given phrase/object pair
adj_matrix[i, :] = cosim
# Save all cosine similarities for all phrases
combo_matrix[old_label_index[old_label] + 1, :, :] = adj_matrix
# Get weighted sum score
score = np.sum(np.multiply(adj_matrix, res_matrix))
# If score is smaller than default score (better!), consider success
# Optional: set minimum difference
if score < default_score: # and ((score-default_score) >=0.1):
better_scores += 1
successful_relations.append(tuple(relation))
# Track and count relations
total_relations.append(tuple(relation))
total_scores += 1
# Track accuracy scores
alignment_acc = get_alignment(adj_matrix, phrase_indices,
sg_object_array, ent_IDs)
sg_alignments_per_rel.append((tuple(relation), alignment_acc))
alignment_accuracies.append(alignment_acc)
# Check whether better/worse/same alignment
if alignment_acc > default_alignment_acc:
better_alignments += 1
if alignment_acc == default_alignment_acc:
same_alignments += 1
total_alignments += 1
# Determine better alignment sucesses
if alignment_acc >= default_alignment_acc:
successful_relations_align.append(tuple(relation))
# Get mean of cosine similarity matrices
adj_matrix = combo_matrix.mean(axis=0)
# Get phrase grounding accuracy
combo_alignment_acc = get_alignment(adj_matrix, phrase_indices,
sg_object_array, ent_IDs)
# Count improvements/non-improvements
if combo_alignment_acc > default_alignment_acc:
combined_better_alignment += 1
elif combo_alignment_acc == default_alignment_acc:
combined_same_alignment += 1
return better_scores, total_scores, successful_relations, total_relations, \
sg_alignments_per_rel, alignment_accuracies, better_alignments, same_alignments, total_alignments, \
combo_alignment_acc, combined_better_alignment, combined_same_alignment, successful_relations_align