def main(arguments):
with open(arguments.filepath, 'r') as fp:
json_exp = json.load(fp)
neptune.init(api_token=arguments.neptune_api_token,
project_qualified_name=arguments.project_name)
with neptune.create_experiment(
name=json_exp['name'],
description=json_exp['description'],
params=json_exp['params'],
properties=json_exp['properties'],
tags=json_exp['tags'],
upload_source_files=json_exp['upload_source_files']):
for name, channel_xy in json_exp['send_metric'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_metric(name, x=x, y=y)
for name, channel_xy in json_exp['send_text'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_text(name, x=x, y=y)
for name, channel_xy in json_exp['send_image'].items():
for x, y in zip(channel_xy['x'], channel_xy['y']):
neptune.send_image(name, x=x, y=y)
def on_epoch_end(self, *args):
self.epoch_counter += 1
if self.epoch_counter % 2 == 0:
self.input_to_message = collections.defaultdict(list)
self.message_to_output = collections.defaultdict(list)
train_state = self.trainer.game.training # persist so we restore it back
self.trainer.game.train(mode=False)
for _ in range(10):
self.run_inference()
self.concept_symbol_matrix, concepts = compute_concept_symbol_matrix(
self.input_to_message,
input_dimensions=[self.opts.n_features] * self.opts.n_attributes,
vocab_size=self.vocab_size
)
self.trainer.game.train(mode=train_state)
self.print_table_input_to_message()
self.draw_concept_symbol_matrix()
# Context independence metrics
context_independence_scores, v_cs = compute_context_independence(
self.concept_symbol_matrix,
input_dimensions=[self.opts.n_features] * self.opts.n_attributes,
)
neptune.send_metric(self.prefix + 'context independence', self.epoch_counter, context_independence_scores.mean(axis=0))
neptune.send_text(self.prefix + 'v_cs', str(v_cs.tolist()))
neptune.send_text(self.prefix + 'context independence scores', str(context_independence_scores.tolist()))
# RSA
correlation_coeff, p_value = compute_representation_similarity(
self.input_to_message,
input_dimensions=[self.opts.n_features] * self.opts.n_attributes
)
neptune.send_metric(self.prefix + 'RSA', self.epoch_counter, correlation_coeff)
neptune.send_metric(self.prefix + 'RSA_p_value', self.epoch_counter, p_value)
def run_inference(self):
with torch.no_grad():
inputs, = self.dataset.tensors
messages, _, _ = self.trainer.game.sender(inputs)
for i in range(inputs.size(0)):
input = tuple(inputs[i].argmax(dim=1).tolist())
message = tuple(prune(messages[i].tolist()))
neptune.send_text(self.prefix + 'messages',
f'{input} -> {message}')
self.input_to_message[input].append(message)
def run_inference(self):
with torch.no_grad():
ran_inference_on = collections.defaultdict(int)
for (input, target) in self.dataset:
target = tuple(target.tolist())
if ran_inference_on[target] < 5:
message = self.sender(input.unsqueeze(dim=0))[0]
message = tuple(message.argmax(dim=1).tolist())
neptune.send_text(self.prefix + 'messages', f'{target} -> {message}')
self.input_to_message[target].append(message)
ran_inference_on[target] += 1
def compute_context_independence(self):
v_cs = self.concept_symbol_matrix.argmax(dim=1)
context_independence_scores = torch.zeros(self.opts.n_features * self.opts.n_attributes)
for concept in range(self.concept_symbol_matrix.size(0)):
v_c = v_cs[concept]
p_vc_c = self.concept_symbol_matrix[concept, v_c]/self.concept_symbol_matrix[concept, :].sum(dim=0)
p_c_vc = self.concept_symbol_matrix[concept, v_c]/self.concept_symbol_matrix[:, v_c].sum(dim=0)
context_independence_scores[concept] = p_vc_c * p_c_vc
neptune.send_text('v_cs', str(v_cs.tolist()))
neptune.send_text('context independence scores', str(context_independence_scores.tolist()))
return context_independence_scores.mean(dim=0)
def on_epoch_end(self, *args):
loader = DataLoader(self.dataset,
batch_size=500,
drop_last=False,
shuffle=True)
input, _ = next(iter(loader))
g1, g2 = self.get_gradients(input)
neptune.send_text('color_grad', str(g1))
neptune.send_text('shape_grad', str(g2))
ax = sns.heatmap(torch.cat([g1, g2], dim=0),
xticklabels=['color', 'shape'],
yticklabels=['$m_1$', '$m_2$'])
figure = ax.get_figure()
send_figure(figure, channel_name=f'{self.label} grads')
figure.savefig('fig.png')
plt.close(figure)
def __init__(self, alpha):
if USE_NEPTUNE and main_process:
neptune.create_experiment('Q learning M alpha = {}'.format(alpha))
self.multi_arena = MultiArena()
self.local_trainer = QLearningTrainer(alpha=alpha)
token = None
if main_process:
token = self.local_trainer._get_token()
print('Current token = {}'.format(token))
if USE_NEPTUNE:
neptune.send_text('weights token', x=token)
self.local_trainer._save_weights()
token = comm.bcast(token, root=0)
self.local_trainer._set_token(token)
def run_inference(self):
with torch.no_grad():
inputs, targets = self.dataset.tensors
indices, _, _ = self.trainer.game.executive_sender(inputs)
messages = self.trainer.game.sender_ensemble(inputs, indices)
receiver_output_1 = self.trainer.game.first_receiver_ensemble(messages, indices)[:, -1, ...]
receiver_output_2 = self.trainer.game.second_receiver_ensemble(messages, indices)[:, -1, ...]
for i in range(inputs.size(0)):
input = tuple(inputs[i].argmax(dim=1).tolist())
index = indices[i].item()
message = tuple(messages[i].argmax(dim=1).tolist())
output = tuple([receiver_output_1[i].argmax(dim=0).item(), receiver_output_2[i].argmax(dim=0).item()])
target = tuple(targets[i].tolist())
if i == 0:
neptune.send_text('messages', f'{input} -> {index} -> {message} -> {output} (expected {target})')
self.counter[Rollout(input, message)] += 1
self.input_to_message[input].append(message)
self.message_to_output[message].append(output)
def monitor(res):
neptune.send_metric('run_score', res.func_vals[-1])
neptune.send_text('run_parameters', str(to_named_params(res.x_iters[-1])))
