def run(host_name='local', pipeline=''):
if pipeline != '':
return exp_shared.load_exp(pipeline)
# Create and run new experiment
exp = Experiment(exp_name='random',
fixed_params=[('env', 'wcs')],
param_ranges=[('avg_over', range(25)),
('term_usage', range(3, 12))])
wcs = com_enviroments.make(exp.fixed_params['env'])
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
N = wcs.data_dim()
map = np.array([
np.random.randint(params_v[exp.axes['term_usage']])
for n in range(N)
])
exp.set_result('language_map', params_i, map)
return exp
def load_exp(pipeline, resync='n'):
exp = Experiment.load(pipeline)
if resync == 'y':
exp.wait(retry_interval=5)
exp.queue.sync(exp.pipeline_path,
exp.pipeline_path,
sync_to=sge.SyncTo.LOCAL,
recursive=True)
return exp
def main():
exp = Experiment.load('color_fix.10')
plot_with_conf2(exp,
'regier_cost',
'term_usage',
'com_noise',
z_label='com $\sigma^2$')
plot_with_conf2(exp, 'gibson_cost', 'term_usage', 'com_noise')
plot_with_conf2(exp, 'wellformedness', 'term_usage', 'com_noise')
def main():
consensus_iters = 10
e = com_enviroments.make('wcs')
k = 3
sims = []
# human maps
human_maps = list(e.human_mode_maps.values())
# robo maps
exp = Experiment.load('color_fix.1')
robo_maps = exp.reshape('agent_language_map')
human_rand = evaluate.mean_rand_index(human_maps)
exp.log.info('mean rand for all human maps = {:.3f}'.format(human_rand))
robo_rand = evaluate.mean_rand_index(robo_maps)
exp.log.info('mean rand for all agent maps = {:.3f}'.format(robo_rand))
cross_rand = evaluate.mean_rand_index(human_maps, robo_maps)
exp.log.info(
'mean rand cross human and robot maps = {:.3f}'.format(cross_rand))
for k in range(3, 12):
cielab_map = evaluate.compute_cielab_map(e,
k,
iter=consensus_iters,
bw_boost=1)
e.plot_with_colors(cielab_map,
save_to_path=exp.pipeline_path + 'cielab_map_' +
str(k) + '.png')
human_consensus_map = Correlation_Clustering.compute_consensus_map(
human_maps, k=k, iter=consensus_iters)
e.plot_with_colors(human_consensus_map,
save_to_path=exp.pipeline_path +
'human_consensus_language_map_' + str(k) + '.png')
robo_consensus_map = Correlation_Clustering.compute_consensus_map(
robo_maps, k=k, iter=consensus_iters)
e.plot_with_colors(robo_consensus_map,
save_to_path=exp.pipeline_path +
'consensus_language_map_' + str(k) + '.png')
# compare human and robo maps
rand_i = adjusted_rand_score(human_consensus_map, robo_consensus_map)
print('rand i between human consensus and agent consensus = {:.3f}'.
format(rand_i))
sims += [rand_i]
sims = np.array(sims)
print(sims.mean())
def run(host_name='local', pipeline=''):
if pipeline != '':
return exp_shared.load_exp(pipeline)
# Create and run new experiment
queue = exp_shared.create_queue(host_name)
queue.sync('.', '.', exclude=['pipelines/*', 'fig/*', 'old/*', 'cogsci/*'], sync_to=sge.SyncTo.REMOTE,
recursive=True)
exp = Experiment(exp_name='ccc',
fixed_params=[('iterations', 10),
('env', 'wcs')],
param_ranges=[('avg_over', range(5)),
('bw_boost', [1]),
('term_usage', range(3, 12))], # np.linspace(start=0, stop=1, num=1)
queue=queue)
queue.sync(exp.pipeline_path, exp.pipeline_path, sync_to=sge.SyncTo.REMOTE, recursive=True)
wcs = com_enviroments.make(exp.fixed_params['env'])
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
N = wcs.data_dim()
corr_graph = np.zeros((N, N))
for i in range(0, N):
for j in range(0, i):
corr_graph[i, j] = (wcs.sim_index(i, j, bw_boost=params_v[exp.axes['bw_boost']]).numpy() - 0.5) * 100
corr_graph[j, i] = (wcs.sim_index(i, j, bw_boost=params_v[exp.axes['bw_boost']]).numpy() - 0.5) * 100
consensus = exp.run(Correlation_Clustering.max_correlation,
corr_graph,
params_v[exp.axes['term_usage']],
exp.fixed_params['iterations']).result()
#print(params_v)
#print('set {} actual {}'.format(params_v[exp.axes['term_usage']], exp.run(evaluate.compute_term_usage, V=consensus).result().get()))
exp.set_result('language_map', params_i, consensus)
return exp
def visualize(pipeline_name):
print('plot results')
exp = Experiment.load(pipeline_name)
viz.plot_with_std(exp,
'gibson_cost', 'com_noise', 'msg_dim',
measure_label='Gibson communication efficiency',
x_label='Communication noise',
z_label='terms')
viz.plot_with_std(exp, 'gibson_cost', 'perception_noise', 'msg_dim')
viz.plot_with_std(exp, 'gibson_cost', 'com_noise', 'perception_noise')
viz.plot_with_std(exp, 'regier_cost', 'com_noise', 'msg_dim')
viz.plot_with_std(exp, 'wellformedness', 'com_noise', 'msg_dim')
def run(host_name='local', pipeline=''):
if pipeline != '':
return exp_shared.load_exp(pipeline)
# Create and run new experiment
queue = exp_shared.create_queue(host_name)
queue.sync('.', '.', exclude=['pipelines/*', 'fig/*', 'old/*', 'cogsci/*'], sync_to=sge.SyncTo.REMOTE,
recursive=True)
exp = Experiment(exp_name='rl',
fixed_params=[('loss_type', 'REINFORCE'),
('bw_boost', 1),
('env', 'wcs'),
('max_epochs', 25000), # 10000
('hidden_dim', 20),
('batch_size', 2048),
('perception_dim', 3),
('target_dim', 330),
('print_interval', 1000),
('msg_dim', 15)],
param_ranges=[('avg_over', range(1)), # 50
('perception_noise',[0, 0.1, 0.3, 0.5, 1, 2, 5]), # np.logspace(0, 9, num=10, base=2)) [0, 10, 20, 30, 40, 50, 80, 120, 160, 320]), [0, 25, 50, 100],[0, 10, 20, 40, 80, 160, 320]
('com_noise', [0, 0.1, 0.3, 0.5, 1, 2, 5])], # np.logspace(-3, 6, num=10, base=2) [0, 0.1, 0.3, 0.5, 1] [0, 0.5, 3, 10, 20, 50]
queue=queue)
queue.sync(exp.pipeline_path, exp.pipeline_path, sync_to=sge.SyncTo.REMOTE, recursive=True)
env = exp.run(com_enviroments.make, exp.fixed_params['env']).result()
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
agent_a = agents.SoftmaxAgent(msg_dim=exp.fixed_params['msg_dim'],
hidden_dim=exp.fixed_params['hidden_dim'],
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
agent_b = agents.SoftmaxAgent(msg_dim=exp.fixed_params['msg_dim'],
hidden_dim=exp.fixed_params['hidden_dim'],
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
game = com_game.NoisyChannelGame(com_noise=params_v[exp.axes['com_noise']],
msg_dim=exp.fixed_params['msg_dim'],
max_epochs=exp.fixed_params['max_epochs'],
perception_noise=params_v[exp.axes['perception_noise']],
batch_size=exp.fixed_params['batch_size'],
print_interval=exp.fixed_params['print_interval'],
loss_type=exp.fixed_params['loss_type'],
bw_boost=exp.fixed_params['bw_boost'])
agent_a_trained = exp.run(game.play, env, agent_a, agent_b).result()
exp.set_result('agent_a', params_i, agent_a_trained)
return exp
def main():
args = exp_shared.parse_script_arguments()
# Run experiment
if args.pipeline == '':
exp = run(args.host_name)
else:
# Load existing experiment
exp = Experiment.load(args.pipeline)
if args.resync == 'y':
exp.wait(retry_interval=5)
exp.queue.sync(exp.pipeline_path, exp.pipeline_path, sync_to=sge.SyncTo.LOCAL, recursive=True)
cluster_ensemble = exp.get_flattened_results('agent_language_map')
consensus = Correlation_Clustering.compute_consensus_map(cluster_ensemble, k=10, iter=100)
print(consensus.values())
# Visualize experiment
visualize(exp)
def run(host_name):
# Create and run new experiment
queue = exp_shared.create_queue(host_name)
queue.sync('.',
'.',
exclude=['pipelines/*', 'fig/*', 'old/*', 'cogsci/*'],
sync_to=sge.SyncTo.REMOTE,
recursive=True)
exp = Experiment(
exp_name='num_b',
fixed_params=[
('env', 'numbers'),
('max_epochs', 1000), #10000
('hidden_dim', 3),
('batch_size', 100),
('perception_dim', 1),
('target_dim', 100),
('print_interval', 10)
],
param_ranges=[
('avg_over', [0]), # 50
('perception_noise', [0]), # [0, 25, 50, 100],
('msg_dim', [3]), #3, 12
('com_noise', [0])
],
queue=queue)
queue.sync(exp.pipeline_path,
exp.pipeline_path,
sync_to=sge.SyncTo.REMOTE,
recursive=True)
env = exp.run(com_enviroments.make, exp.fixed_params['env']).result()
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
#print('Param epoch %d of %d' % (params_i[exp.axes['avg_over']], exp.shape[exp.axes['avg_over']]))
agent_a = agents.SoftmaxAgent(
msg_dim=params_v[exp.axes['msg_dim']],
hidden_dim=exp.fixed_params['hidden_dim'],
# shared_dim=3,
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
agent_b = agents.SoftmaxAgent(
msg_dim=params_v[exp.axes['msg_dim']],
hidden_dim=exp.fixed_params['hidden_dim'],
# shared_dim=3,
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
game = com_game.NoisyChannelGame(
reward_func='abs_dist',
com_noise=params_v[exp.axes['com_noise']],
msg_dim=params_v[exp.axes['msg_dim']],
max_epochs=exp.fixed_params['max_epochs'],
perception_noise=params_v[exp.axes['perception_noise']],
batch_size=exp.fixed_params['batch_size'],
print_interval=exp.fixed_params['print_interval'],
perception_dim=exp.fixed_params['perception_dim'],
loss_type='REINFORCE')
game_outcome = exp.run(game.play, env, agent_a, agent_b).result()
V = exp.run(evaluate.agent_language_map, env, a=game_outcome).result()
exp.set_result('agent_language_map', params_i, V)
exp.set_result(
'gibson_cost', params_i,
exp.run(game.compute_gibson_cost, env, a=game_outcome).result(1))
exp.set_result(
'regier_cost', params_i,
exp.run(evaluate.communication_cost_regier, env, V=V).result())
exp.set_result('wellformedness', params_i,
exp.run(evaluate.wellformedness, env, V=V).result())
exp.set_result('term_usage', params_i,
exp.run(evaluate.compute_term_usage, V=V).result())
print("\nAll tasks queued to clusters")
# wait for all tasks to complete
exp.save()
exp.wait(retry_interval=5)
queue.sync(exp.pipeline_path,
exp.pipeline_path,
sync_to=sge.SyncTo.LOCAL,
recursive=True)
return exp
def run():
exp = Experiment(exp_name='local_experiment',
fixed_params=[('env', 'wgs'),
('max_epochs', 10000), #10000
('hidden_dim', 20),
('batch_size', 100),
('perception_dim', 3),
('target_dim', 330),
('print_interval', 1000)],
param_ranges=[('avg_over', range(2)), # 50
('perception_noise', [0, 25]), # [0, 25, 50, 100],
('msg_dim', range(9, 11)), #3, 12
('com_noise', np.linspace(start=0, stop=0.5, num=2))])
env = com_enviroments.make(exp.fixed_params['env'])
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
agent_a = agent_b = agents.SoftmaxAgent(msg_dim=params_v[exp.axes['msg_dim']],
hidden_dim=exp.fixed_params['hidden_dim'],
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
game = com_game.NoisyChannelGame(com_noise=params_v[exp.axes['com_noise']],
msg_dim=params_v[exp.axes['msg_dim']],
max_epochs=exp.fixed_params['max_epochs'],
perception_noise=params_v[exp.axes['perception_noise']],
batch_size=exp.fixed_params['batch_size'],
print_interval=exp.fixed_params['print_interval'])
game_outcome = game.play(env, agent_a, agent_b)
V = evaluate.agent_language_map(env, a=game_outcome)
exp.set_result('gibson_cost', params_i, game.compute_gibson_cost(env, a=game_outcome)[1])
exp.set_result('regier_cost', params_i, evaluate.communication_cost_regier(env, V=V))
exp.set_result('wellformedness', params_i, evaluate.wellformedness(env, V=V))
exp.set_result('term_usage', params_i, evaluate.compute_term_usage(V=V))
print("\nAll tasks queued to clusters")
# wait for all tasks to complete
exp.save()
return exp.pipeline_name
def run(host_name='local', pipeline=''):
if pipeline != '':
return exp_shared.load_exp(pipeline)
wcs = com_enviroments.make('wcs')
# Create and run new experiment
exp = Experiment(exp_name='human',
fixed_params=[('env', 'wcs')],
param_ranges=[
('lang_id', list(wcs.human_mode_maps.keys()))
]) # range(1, 5))]) #range(1, 5))]) #
exp_i = 0
for (params_i, params_v) in exp:
print('Scheduled %d experiments out of %d' % (exp_i, len(list(exp))))
exp_i += 1
map = wcs.human_mode_maps[params_v[exp.axes['lang_id']]]
exp.set_result('language_map', params_i, map)
exp.set_result('term_usage', params_i,
exp.run(evaluate.compute_term_usage, V=map).result())
exp.set_result('regier_cost', params_i,
exp.run(evaluate.regier2, wcs, map=map).result())
#exp.set_result('regier_cost', params_i, exp.run(evaluate.communication_cost_regier, wcs, V=map, sum_over_whole_s=True).result())
exp.set_result('wellformedness', params_i,
exp.run(evaluate.wellformedness, wcs, V=map).result())
exp.save()
return exp
def run(host_name='local', pipeline=''):
if pipeline != '':
return exp_shared.load_exp(pipeline)
# Create and run new experiment
queue = exp_shared.create_queue(host_name)
queue.sync('.',
'.',
exclude=['pipelines/*', 'fig/*', 'old/*', 'cogsci/*'],
sync_to=sge.SyncTo.REMOTE,
recursive=True)
exp = Experiment(
exp_name='rl_evo_dev',
fixed_params=[
('loss_type', 'REINFORCE'),
('bw_boost', 1),
('env', 'wcs'),
('max_epochs', 50000), # 10000
('hidden_dim', 20),
('batch_size', 100),
('perception_dim', 3),
('target_dim', 330),
('print_interval', 1000),
('evaluate_interval', 100),
('msg_dim', 15),
('com_noise', 0.1),
('perception_noise', 40)
], #[0, 10, 20, 30, 40, 50, 80, 120, 160, 320]
queue=queue)
queue.sync(exp.pipeline_path,
exp.pipeline_path,
sync_to=sge.SyncTo.REMOTE,
recursive=True)
env = exp.run(com_enviroments.make, exp.fixed_params['env']).result()
agent_a = agents.SoftmaxAgent(
msg_dim=exp.fixed_params['msg_dim'],
hidden_dim=exp.fixed_params['hidden_dim'],
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
agent_b = agents.SoftmaxAgent(
msg_dim=exp.fixed_params['msg_dim'],
hidden_dim=exp.fixed_params['hidden_dim'],
color_dim=exp.fixed_params['target_dim'],
perception_dim=exp.fixed_params['perception_dim'])
game = com_game.NoisyChannelGame(
com_noise=exp.fixed_params['com_noise'],
msg_dim=exp.fixed_params['msg_dim'],
max_epochs=exp.fixed_params['max_epochs'],
perception_noise=exp.fixed_params['perception_noise'],
batch_size=exp.fixed_params['batch_size'],
print_interval=exp.fixed_params['print_interval'],
evaluate_interval=exp.fixed_params['evaluate_interval'],
loss_type=exp.fixed_params['loss_type'],
bw_boost=exp.fixed_params['bw_boost'],
log_path=exp.pipeline_path)
game_outcome = exp.run(game.play, env, agent_a, agent_b).result()
V = exp.run(evaluate.agent_language_map, env, a=game_outcome).result()
# exp.set_result('agent_language_map', V)
# exp.set_result('gibson_cost', exp.run(game.compute_gibson_cost, env, a=game_outcome).result(1))
# exp.set_result('regier_cost', exp.run(evaluate.communication_cost_regier, env, V=V).result())
# exp.set_result('wellformedness', exp.run(evaluate.wellformedness, env, V=V).result())
# exp.set_result('term_usage', exp.run(evaluate.compute_term_usage, V=V).result())
exp.save()
print("\nAll tasks queued to clusters")
# wait for all tasks to complete
exp.wait(retry_interval=5)
queue.sync(exp.pipeline_path,
exp.pipeline_path,
sync_to=sge.SyncTo.LOCAL,
recursive=True)
return exp