def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=4)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
report.save()
inner_env = normalize(Arm3dDiscEnv())
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
fixed_start_generator = FixedStateGenerator(state=v['ultimate_goal'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=fixed_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[-1 * v['goal_size']:],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
inner_weight=v['inner_weight'],
goal_weight=v['goal_weight'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# load the state collection from data_upload
load_dir = 'data_upload/state_collections/'
all_feasible_starts = pickle.load(
open(
osp.join(config.PROJECT_PATH, load_dir,
'disc_all_feasible_states_min.pkl'), 'rb'))
print("we have %d feasible starts" % all_feasible_starts.size)
all_starts = StateCollection(distance_threshold=v['coll_eps'])
# brownian_starts = StateCollection(distance_threshold=v['regularize_starts'])
# with env.set_kill_outside():
# seed_starts = generate_starts(env, starts=[v['start_goal']], horizon=10, # this is smaller as they are seeds!
# variance=v['brownian_variance'], subsample=v['num_new_starts']) # , animated=True, speedup=1)
#
# with env.set_kill_outside():
# find_all_feasible_states(env, seed_starts, distance_threshold=0.1, brownian_variance=1, animate=False)
# show where these states are:
# shuffled_starts = np.array(all_feasible_starts.state_list)
# np.random.shuffle(shuffled_starts)
# generate_starts(env, starts=shuffled_starts, horizon=100, variance=v['brownian_variance'], animated=True, speedup=10)
# Use asymmetric self-play to run Alice to generate starts for Bob.
env_alice = AliceEnv(env, env, policy, v['horizon'])
policy_alice = GaussianMLPPolicy(
env_spec=env_alice.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain_alice'],
init_std=v['policy_init_std_alice'],
)
baseline_alice = LinearFeatureBaseline(env_spec=env_alice.spec)
algo_alice = TRPO(
env=env_alice,
policy=policy_alice,
baseline=baseline_alice,
batch_size=v['pg_batch_size_alice'],
max_path_length=v['horizon'],
n_itr=v['inner_iters_alice'],
step_size=0.01,
discount=v['discount_alice'],
plot=False,
)
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
# with env.set_kill_outside():
# starts = generate_starts(env, starts=seed_starts, horizon=v['brownian_horizon'], variance=v['brownian_variance'])
# regularization of the brownian starts
# brownian_starts.empty()
# brownian_starts.append(starts)
# starts = brownian_starts.sample(size=v['num_new_starts'])
starts = generate_starts_alice(env_bob=env,
env_alice=env_alice,
policy_bob=policy,
policy_alice=policy_alice,
algo_alice=algo_alice,
start_states=[v['start_goal']],
num_new_starts=v['num_new_starts'],
alice_factor=v['alice_factor'],
log_dir=log_dir)
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
discount=v['discount'],
plot=False,
)
trpo_paths = algo.train()
if v['use_trpo_paths']:
logger.log("labeling starts with trpo rollouts")
[starts, labels] = label_states_from_paths(
trpo_paths,
n_traj=2,
key='goal_reached', # using the min n_traj
as_goal=False,
env=env)
paths = [path for paths in trpo_paths for path in paths]
else:
logger.log("labeling starts manually")
labels, paths = label_states(starts,
env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached',
full_path=True)
with logger.tabular_prefix("OnStarts_"):
env.log_diagnostics(paths)
logger.record_tabular('starts', starts.size)
start_classes, text_labels = convert_label(labels)
total_starts = labels.shape[0]
logger.record_tabular('GenStarts_evaluated', total_starts)
start_class_frac = OrderedDict(
) # this needs to be an ordered dict!! (for the log tabular)
for k in text_labels.keys():
frac = np.sum(start_classes == k) / total_starts
logger.record_tabular('GenStart_frac_' + text_labels[k], frac)
start_class_frac[text_labels[k]] = frac
labels = np.logical_and(labels[:, 0],
labels[:, 1]).astype(int).reshape((-1, 1))
logger.log("Labeling on uniform starts")
with logger.tabular_prefix("Uniform_"):
unif_starts = all_feasible_starts.sample(1000)
mean_reward, paths = evaluate_states(unif_starts,
env,
policy,
v['horizon'],
n_traj=1,
key='goal_reached',
as_goals=False,
full_path=True)
env.log_diagnostics(paths)
logger.dump_tabular(with_prefix=True)
# append new states to list of all starts (replay buffer): Not the low reward ones!!
logger.log("Appending good goals to replay and generating seeds")
filtered_raw_starts = [
start for start, label in zip(starts, labels) if label[0] == 1
]
all_starts.append(filtered_raw_starts)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
samples_per_cell = 10 # for the oracle rejection sampling
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=5)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(PointMazeEnv(maze_id=v['maze_id']))
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
uniform_start_generator = UniformStateGenerator(state_size=v['start_size'], bounds=v['start_range'],
center=v['start_center'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=uniform_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[:v['goal_size']],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16, 16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
if v['constant_baseline']:
logger.log("Using constant baseline")
baseline = ConstantBaseline(env_spec=env.spec, value=1.0)
else:
logger.log("Using linear baseline")
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
plot_policy_means(policy, env, sampling_res=2, report=report, limit=v['goal_range'], center=v['goal_center'])
test_and_plot_policy(policy, env, as_goals=False, max_reward=v['max_reward'], sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter, report=report, center=v['goal_center'],
limit=v['goal_range']) # use goal for plot
report.new_row()
all_starts = StateCollection(distance_threshold=v['coll_eps'])
seed_starts = generate_starts(env, starts=[v['ultimate_goal']], subsample=v['num_new_starts'])
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
starts = generate_starts(env, starts=seed_starts, subsample=v['num_new_starts'],
horizon=v['brownian_horizon'], variance=v['brownian_variance'])
labels = label_states(starts, env, policy, v['horizon'],
as_goals=False, n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(starts, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'],
summary_string_base='initial starts labels:\n')
report.save()
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
with ExperimentLogger(log_dir, 'last', snapshot_mode='last', hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(), persistence=v['persistence'], with_replacement=v['with_replacement'],
)
)
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
discount=v['discount'],
plot=False,
)
trpo_paths = algo.train()
if v['use_trpo_paths']:
logger.log("labeling starts with trpo rollouts")
[starts, labels] = label_states_from_paths(trpo_paths, n_traj=2, key='goal_reached', # using the min n_traj
as_goal=False, env=env)
paths = [path for paths in trpo_paths for path in paths]
else:
logger.log("labeling starts manually")
labels, paths = label_states(starts, env, policy, v['horizon'], as_goals=False, n_traj=v['n_traj'],
key='goal_reached', full_path=True)
with logger.tabular_prefix("OnStarts_"):
env.log_diagnostics(paths)
logger.log('Generating the Heatmap...')
plot_policy_means(policy, env, sampling_res=2, report=report, limit=v['goal_range'], center=v['goal_center'])
test_and_plot_policy(policy, env, as_goals=False, max_reward=v['max_reward'], sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter, report=report, center=v['goal_center'], limit=v['goal_range'])
logger.log("Labeling the starts")
#labels = label_states(starts, env, policy, v['horizon'], as_goals=False, n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(starts, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'])
start_classes, text_labels = convert_label(labels)
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
labels = np.logical_and(labels[:, 0], labels[:, 1]).astype(int).reshape((-1, 1))
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new states to list of all starts (replay buffer): Not the low reward ones!!
filtered_raw_starts = [start for start, label in zip(starts, labels) if label[0] == 1]
all_starts.append(filtered_raw_starts)
if v['seed_with'] == 'only_goods':
if len(filtered_raw_starts) > 0: # add a tone of noise if all the states I had ended up being high_reward!
seed_starts = filtered_raw_starts
elif np.sum(start_classes == 0) > np.sum(start_classes == 1): # if more low reward than high reward
seed_starts = all_starts.sample(300) # sample them from the replay
else:
seed_starts = generate_starts(env, starts=starts, horizon=int(v['horizon'] * 10), subsample=v['num_new_starts'],
variance=v['brownian_variance'] * 10)
elif v['seed_with'] == 'all_previous':
seed_starts = starts
elif v['seed_with'] == 'on_policy':
seed_starts = generate_starts(env, policy, starts=starts, horizon=v['horizon'], subsample=v['num_new_starts'])
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
if log_dir is None:
log_dir = "/home/michael/"
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=4)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(SwimmerMazeEnv())
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
fixed_start_generator = FixedStateGenerator(state=v['ultimate_goal'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=fixed_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[-3:-1],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
inner_weight=v['inner_weight'],
goal_weight=v['goal_weight'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# load the state collection from data_upload
load_dir = 'data_upload/state_collections/'
# all_feasible_starts = pickle.load(open(osp.join(config.PROJECT_PATH, load_dir, 'all_feasible_states_min.pkl'), 'rb'))
# print("we have %d feasible starts" % all_feasible_starts.size)
all_starts = StateCollection(distance_threshold=v['coll_eps'])
# brownian_starts = StateCollection(distance_threshold=v['regularize_starts'])
# with env.set_kill_outside():
seed_starts = generate_starts(
env,
starts=[v['start_goal']],
horizon=v['initial_brownian_horizon'],
size=5000, # size speeds up training a bit
variance=v['brownian_variance'],
subsample=v['num_new_starts']) # , animated=True, speedup=1)
np.random.shuffle(seed_starts)
# with env.set_kill_outside():
feasible_states = find_all_feasible_states_plotting(env,
seed_starts,
distance_threshold=1,
brownian_variance=1,
animate=True)
# print("hi")
# show where these states are:
# shuffled_starts = np.array(seed_starts.state_list)
# np.random.shuffle(shuffled_starts)
# generate_starts(env, starts=seed_starts, horizon=100, variance=v['brownian_variance'], animated=True, speedup=10)
if 'gae_lambda' not in v:
v['gae_lambda'] = 1
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
starts = generate_starts(env,
starts=seed_starts,
subsample=v['num_new_starts'],
size=5000,
horizon=v['brownian_horizon'],
variance=v['brownian_variance'])
labels = label_states(starts,
env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(starts,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'],
summary_string_base='initial starts labels:\n')
report.save()
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
gae_lambda=v['gae_lambda'],
step_size=0.01,
discount=v['discount'],
plot=False,
)
algo.train()
logger.log('Generating the Heatmap...')
# policy means should not mean too much
# plot_policy_means(policy, env, sampling_res=2, report=report, limit=v['goal_range'], center=v['goal_center'])
test_and_plot_policy(policy,
env,
as_goals=False,
max_reward=v['max_reward'],
sampling_res=1,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
center=v['goal_center'],
limit=v['goal_range'])
logger.log("Labeling the starts")
labels = label_states(starts,
env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(starts,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
labels = np.logical_and(labels[:, 0],
labels[:, 1]).astype(int).reshape((-1, 1))
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new states to list of all starts (replay buffer): Not the low reward ones!!
filtered_raw_starts = [
start for start, label in zip(starts, labels) if label[0] == 1
]
if len(
filtered_raw_starts
) > 0: # add a tone of noise if all the states I had ended up being high_reward!
seed_starts = filtered_raw_starts
else:
seed_starts = generate_starts(env,
starts=starts,
horizon=v['horizon'] * 2,
subsample=v['num_new_starts'],
size=5000,
variance=v['brownian_variance'] * 10)
all_starts.append(filtered_raw_starts)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=3)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
tf_session = tf.Session()
inner_env = normalize(AntMazeEnv())
uniform_goal_generator = UniformStateGenerator(state_size=v['goal_size'], bounds=v['goal_range'],
center=v['goal_center'])
env = GoalExplorationEnv(
env=inner_env, goal_generator=uniform_goal_generator,
obs2goal_transform=lambda x: x[-3:-1],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16, 16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
test_and_plot_policy(policy, env, max_reward=v['max_reward'], sampling_res=sampling_res, n_traj=v['n_traj'],
itr=outer_iter, report=report, limit=v['goal_range'], center=v['goal_center'])
# GAN
logger.log("Instantiating the GAN...")
gan_configs = {key[4:]: value for key, value in v.items() if 'GAN_' in key}
for key, value in gan_configs.items():
if value is tf.train.AdamOptimizer:
gan_configs[key] = tf.train.AdamOptimizer(gan_configs[key + '_stepSize'])
if value is tflearn.initializations.truncated_normal:
gan_configs[key] = tflearn.initializations.truncated_normal(stddev=gan_configs[key + '_stddev'])
gan = StateGAN(
state_size=v['goal_size'],
evaluater_size=v['num_labels'],
state_range=v['goal_range'],
state_center=v['goal_center'],
state_noise_level=v['goal_noise_level'],
generator_layers=v['gan_generator_layers'],
discriminator_layers=v['gan_discriminator_layers'],
noise_size=v['gan_noise_size'],
tf_session=tf_session,
configs=gan_configs,
)
logger.log("pretraining the GAN...")
if v['smart_init']:
feasible_goals = generate_initial_goals(env, policy, v['goal_range'], goal_center=v['goal_center'],
horizon=v['horizon'])
labels = np.ones((feasible_goals.shape[0], 2)).astype(np.float32) # make them all good goals
plot_labeled_states(feasible_goals, labels, report=report, itr=outer_iter,
limit=v['goal_range'], center=v['goal_center'])
dis_loss, gen_loss = gan.pretrain(states=feasible_goals, outer_iters=v['gan_outer_iters'])
print("Loss of Gen and Dis: ", gen_loss, dis_loss)
else:
gan.pretrain_uniform()
# log first samples form the GAN
initial_goals, _ = gan.sample_states_with_noise(v['num_new_goals'])
logger.log("Labeling the goals")
labels = label_states(initial_goals, env, policy, v['horizon'], n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(initial_goals, labels, report=report, itr=outer_iter,
limit=v['goal_range'], center=v['goal_center'])
report.new_row()
all_goals = StateCollection(distance_threshold=v['coll_eps'])
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
# Sample GAN
logger.log("Sampling goals from the GAN")
raw_goals, _ = gan.sample_states_with_noise(v['num_new_goals'])
if v['replay_buffer'] and outer_iter > 0 and all_goals.size > 0:
old_goals = all_goals.sample(v['num_old_goals'])
goals = np.vstack([raw_goals, old_goals])
else:
goals = raw_goals
# if needed label the goals before any update
if v['label_with_variation']:
old_labels, old_rewards = label_states(goals, env, policy, v['horizon'], as_goals=True, n_traj=v['n_traj'],
key='goal_reached', full_path=False, return_rew=True)
# itr_label = outer_iter # use outer_iter to log everything or "last" to log only the last
# with ExperimentLogger(log_dir, itr_label, snapshot_mode='last', hold_outter_log=True):
with ExperimentLogger(log_dir, 'last', snapshot_mode='last', hold_outter_log=True):
logger.log("Updating the environment goal generator")
env.update_goal_generator(
UniformListStateGenerator(
goals.tolist(), persistence=v['persistence'], with_replacement=v['with_replacement'],
)
)
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
plot=False,
)
trpo_paths = algo.train()
if v['use_trpo_paths']:
logger.log("labeling starts with trpo rollouts")
[goals, labels] = label_states_from_paths(trpo_paths, n_traj=2, key='goal_reached', # using the min n_traj
as_goal=True, env=env)
paths = [path for paths in trpo_paths for path in paths]
elif v['label_with_variation']:
labels, paths = label_states(goals, env, policy, v['horizon'], as_goals=True, n_traj=v['n_traj'],
key='goal_reached', old_rewards=old_rewards, full_path=True)
else:
logger.log("labeling starts manually")
labels, paths = label_states(goals, env, policy, v['horizon'], as_goals=True, n_traj=v['n_traj'],
key='goal_reached', full_path=True)
with logger.tabular_prefix("OnStarts_"):
env.log_diagnostics(paths)
logger.log('Generating the Heatmap...')
test_and_plot_policy(policy, env, max_reward=v['max_reward'], sampling_res=sampling_res, n_traj=v['n_traj'],
itr=outer_iter, report=report, limit=v['goal_range'], center=v['goal_center'])
#logger.log("Labeling the goals")
#labels = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(goals, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
if v['label_with_variation']: # this will use only the performance variation for labeling
labels = np.array(labels[:, -1], dtype=int).reshape((-1, 1))
else:
labels = np.logical_and(labels[:, 0], labels[:, 1]).astype(int).reshape((-1, 1))
logger.log("Training the GAN")
gan.train(
goals, labels,
v['gan_outer_iters'],
)
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new goals to list of all goals (replay buffer): Not the low reward ones!!
filtered_raw_goals = [goal for goal, label in zip(goals, labels) if label[0] == 1]
all_goals.append(filtered_raw_goals)
if v['add_on_policy']:
logger.log("sampling on policy")
feasible_goals = generate_initial_goals(env, policy, v['goal_range'], goal_center=v['goal_center'],
horizon=v['horizon'])
# downsampled_feasible_goals = feasible_goals[np.random.choice(feasible_goals.shape[0], v['add_on_policy']),:]
all_goals.append(feasible_goals)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
samples_per_cell = 10 # for the oracle rejection sampling
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=3)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(PointMazeEnv(maze_id=v['maze_id']))
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
uniform_start_generator = UniformStateGenerator(state_size=v['start_size'], bounds=v['start_range'],
center=v['start_center'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=uniform_start_generator,
goal_generator=fixed_goal_generator,
obs2start_transform=lambda x: x[:v['start_size']],
obs2goal_transform=lambda x: x[:v['goal_size']],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16, 16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
plot_policy_means(policy, env, sampling_res=2, report=report, limit=v['start_range'], center=v['start_center'])
test_and_plot_policy(policy, env, as_goals=False, max_reward=v['max_reward'], sampling_res=sampling_res, n_traj=v['n_traj'],
itr=outer_iter, report=report, center=v['start_center'], limit=v['start_range'])
report.new_row()
all_starts = StateCollection(distance_threshold=v['coll_eps'])
total_rollouts = 0
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
starts = np.array([]).reshape((-1, v['start_size']))
k = 0
while starts.shape[0] < v['num_new_starts']:
print('good starts collected: ', starts.shape[0])
logger.log("Sampling and labeling the starts: %d" % k)
k += 1
unif_starts = sample_unif_feas(env, samples_per_cell=samples_per_cell)
if v['start_size'] > 2:
unif_starts = np.array([np.concatenate([start, np.random.uniform(-v['start_range'], v['start_range'], 2)])
for start in unif_starts])
labels = label_states(unif_starts, env, policy, v['horizon'],
as_goals=False, n_traj=v['n_traj'], key='goal_reached')
# plot_labeled_states(unif_starts, labels, report=report, itr=outer_iter, limit=v['start_range'],
# center=v['start_center'], maze_id=v['maze_id'])
logger.log("Converting the labels")
init_classes, text_labels = convert_label(labels)
starts = np.concatenate([starts, unif_starts[init_classes == 2]]).reshape((-1, v['start_size']))
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
# report.new_row()
with ExperimentLogger(log_dir, 'last', snapshot_mode='last', hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(), persistence=v['persistence'], with_replacement=v['with_replacement'],
)
)
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
discount=v['discount'],
gae_lambda=v['gae_lambda'],
plot=False,
)
algo.train()
logger.log('Generating the Heatmap...')
plot_policy_means(policy, env, sampling_res=2, report=report, limit=v['start_range'], center=v['start_center'])
test_and_plot_policy(policy, env, as_goals=False, max_reward=v['max_reward'], sampling_res=sampling_res, n_traj=v['n_traj'],
itr=outer_iter, report=report, center=v['goal_center'], limit=v['goal_range'])
logger.log("Labeling the starts")
labels = label_states(starts, env, policy, v['horizon'], as_goals=False, n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(starts, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
labels = np.logical_and(labels[:, 0], labels[:, 1]).astype(int).reshape((-1, 1))
# rollouts used for labeling (before TRPO itrs):
num_empty_spaces = len(unwrap_maze(env).find_empty_space())
logger.record_tabular('LabelingRollouts', k * v['n_traj'] * samples_per_cell * num_empty_spaces)
total_rollouts += k * v['n_traj'] * samples_per_cell * num_empty_spaces
logger.record_tabular('TotalLabelingRollouts', total_rollouts)
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new goals to list of all goals (replay buffer): Not the low reward ones!!
filtered_raw_starts = [start for start, label in zip(starts, labels) if label[0] == 1]
all_starts.append(filtered_raw_starts)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 0 if 'sampling_res' not in v.keys() else v['sampling_res']
unif_samples = 300
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=3)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(AntEnv())
uniform_goal_generator = UniformStateGenerator(state_size=v['goal_size'],
bounds=v['goal_range'],
center=v['goal_center'])
env = GoalExplorationEnv(
env=inner_env,
goal_generator=uniform_goal_generator,
obs2goal_transform=lambda x: x[-3:-1],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
append_transformed_obs=v['append_transformed_obs'],
append_extra_info=v['append_extra_info'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
if v['baseline'] == 'g_mlp':
baseline = GaussianMLPBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'],
bounds=v['goal_range'])
report.new_row()
all_goals = StateCollection(distance_threshold=v['coll_eps'])
total_rollouts = 0
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling goals")
goals = np.array([]).reshape((-1, v['goal_size']))
k = 0
while goals.shape[0] < v['num_new_goals']:
print('good goals collected: ', goals.shape[0])
logger.log("Sampling and labeling the goals: %d" % k)
k += 1
unif_goals = np.random.uniform(
np.array(v['goal_center']) - np.array(v['goal_range']),
np.array(v['goal_center']) + np.array(v['goal_range']),
size=(unif_samples, v['goal_size']))
labels = label_states(unif_goals,
env,
policy,
v['horizon'],
n_traj=v['n_traj'],
key='goal_reached')
logger.log("Converting the labels")
init_classes, text_labels = convert_label(labels)
goals = np.concatenate([goals,
unif_goals[init_classes == 2]]).reshape(
(-1, v['goal_size']))
if v['replay_buffer'] and outer_iter > 0 and all_goals.size > 0:
old_goals = all_goals.sample(
v['num_old_goals']) #todo: replay noise?
goals = np.vstack([goals, old_goals])
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment goal generator")
env.update_goal_generator(
UniformListStateGenerator(
goals.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
plot=False,
)
trpo_paths = algo.train()
logger.log("labeling starts with trpo rollouts")
[goals, labels] = label_states_from_paths(
trpo_paths,
n_traj=2,
key='goal_reached', # using the min n_traj
as_goal=True,
env=env)
paths = [path for paths in trpo_paths for path in paths]
with logger.tabular_prefix("OnStarts_"):
env.log_diagnostics(paths)
logger.log('Generating the Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'],
bounds=v['goal_range'])
plot_labeled_states(goals,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
labels = np.logical_and(labels[:, 0],
labels[:, 1]).astype(int).reshape((-1, 1))
# rollouts used for labeling (before TRPO itrs):
logger.record_tabular('LabelingRollouts',
k * v['n_traj'] * unif_samples)
total_rollouts += k * v['n_traj'] * unif_samples
logger.record_tabular('TotalLabelingRollouts', total_rollouts)
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new goals to list of all goals (replay buffer): Not the low reward ones!!
filtered_raw_goals = [
goal for goal, label in zip(goals, labels) if label[0] == 1
]
all_goals.append(filtered_raw_goals)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir()
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=1000)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(PointMazeEnv(maze_id=v['maze_id']))
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
uniform_start_generator = UniformStateGenerator(state_size=v['start_size'], bounds=v['start_range'],
center=v['start_center'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=uniform_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[:v['goal_size']],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16, 16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
if v["baseline"] == "MLP":
baseline = GaussianMLPBaseline(env_spec=env.spec)
else:
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
all_starts = StateCollection(distance_threshold=v['coll_eps'])
# seed_starts: from which we will be performing brownian motion exploration
seed_starts = generate_starts(env, starts=[v['ultimate_goal']], subsample=v['num_new_starts'])
def plot_states(states, report, itr, summary_string, **kwargs):
states = np.array(states)
if states.size == 0:
states = np.zeros((1, 2))
img = plot_labeled_samples(
states, np.zeros(len(states), dtype='uint8'), markers={0: 'o'}, text_labels={0: "all"}, **kwargs)
report.add_image(img, 'itr: {}\n{}'.format(itr, summary_string), width=500)
for outer_iter in range(1, v['outer_iters']):
report.new_row()
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
plot_states(
seed_starts, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'],
maze_id=v['maze_id'], summary_string="seed starts")
starts = generate_starts(env, starts=seed_starts, subsample=v['num_new_starts'],
horizon=v['brownian_horizon'], variance=v['brownian_variance'])
plot_states(
starts, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'],
maze_id=v['maze_id'], summary_string="brownian starts")
sampled_from_buffer = []
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
sampled_from_buffer = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, sampled_from_buffer])
plot_states(
sampled_from_buffer, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'], summary_string="states sampled from buffer")
labels = label_states(starts, env, policy, v['horizon'], as_goals=False, n_traj=v['n_traj'], key='goal_reached')
plot_labeled_states(starts, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'],
summary_string_base='all starts before update\n')
with ExperimentLogger(log_dir, 'last', snapshot_mode='last', hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(), persistence=v['persistence'], with_replacement=v['with_replacement'],
)
)
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
discount=v['discount'],
plot=False,
)
trpo_paths = algo.train()
if v['use_trpo_paths']:
logger.log("labeling starts with trpo rollouts")
[starts, labels] = label_states_from_paths(
trpo_paths, n_traj=2, key='goal_reached', as_goal=False, env=env)
paths = [path for paths in trpo_paths for path in paths]
else:
logger.log("labeling starts manually")
labels, paths = label_states(
starts, env, policy, v['horizon'], as_goals=False, n_traj=v['n_traj'], key='goal_reached', full_path=True)
start_classes, text_labels = convert_label(labels)
plot_labeled_states(starts, labels, report=report, itr=outer_iter, limit=v['goal_range'],
center=v['goal_center'], maze_id=v['maze_id'],
summary_string_base="all starts after update\n")
with logger.tabular_prefix("OnStarts_"):
env.log_diagnostics(paths)
labels = np.logical_and(labels[:, 0], labels[:, 1]).astype(int).reshape((-1, 1))
# append new states to list of all starts (replay buffer): Not the low reward ones!!
filtered_raw_starts = [start for start, label in zip(starts, labels) if label[0] == 1]
all_starts.append(filtered_raw_starts)
if v['seed_with'] == 'only_goods':
if len(filtered_raw_starts) > 0:
logger.log("Only goods A")
seed_starts = filtered_raw_starts
elif np.sum(start_classes == 0) > np.sum(start_classes == 1): # if more low reward than high reward
logger.log("Only goods B")
seed_starts = all_starts.sample(300) # sample them from the replay
else:
logger.log("Only goods C")
# add a ton of noise if all the states I had ended up being high_reward
seed_starts = generate_starts(
env, starts=starts, horizon=int(v['horizon'] * 10),
subsample=v['num_new_starts'], variance=v['brownian_variance'] * 10)
elif v['seed_with'] == 'all_previous':
seed_starts = starts
elif v['seed_with'] == 'on_policy':
seed_starts = generate_starts(env, policy, starts=starts, horizon=v['horizon'], subsample=v['num_new_starts'])
logger.log('Generating Heatmap...')
plot_policy_means(
policy, env, sampling_res=sampling_res, report=report, limit=v['goal_range'], center=v['goal_center'])
_, _, states, returns, successes = test_and_plot_policy2(
policy, env, as_goals=False, max_reward=v['max_reward'], sampling_res=sampling_res, n_traj=v['n_traj'],
itr=outer_iter, report=report, center=v['goal_center'], limit=v['goal_range'])
eval_state_path = osp.join(log_dir, "eval_states.json")
if not osp.exists(eval_state_path):
with open(eval_state_path, 'w') as f:
json.dump(np.array(states).tolist(), f)
with open(osp.join(log_dir, 'eval_pos_per_state_mean_return.csv'), 'a') as f:
writer = csv.writer(f)
row = [outer_iter] + list(returns)
writer.writerow(row)
with open(osp.join(log_dir, 'eval_pos_per_state_mean_success.csv'), 'a') as f:
writer = csv.writer(f)
row = [outer_iter] + list(successes)
writer.writerow(row)
logger.dump_tabular()
report.save()
if outer_iter == 1 or outer_iter % 5 == 0 and v.get('scratch_dir', False):
command = 'rsync -a {} {}'.format(os.path.join(log_dir, ''), os.path.join(v['scratch_dir'], ''))
print("Running command:\n{}".format(command))
subprocess.run(command.split(), check=True)
if v.get('scratch_dir', False):
command = 'rsync -a {} {}'.format(os.path.join(log_dir, ''), os.path.join(v['scratch_dir'], ''))
print("Running command:\n{}".format(command))
subprocess.run(command.split(), check=True)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=4)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(Arm3dKeyEnv(ctrl_cost_coeff=v['ctrl_cost_coeff']))
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
fixed_start_generator = FixedStateGenerator(state=v['start_goal'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=fixed_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[-1 * v['goal_size']:
], # the goal are the last 9 coords
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
inner_weight=v['inner_weight'],
goal_weight=v['goal_weight'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=v['policy_hidden_sizes'],
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
if v['baseline'] == 'linear':
baseline = LinearFeatureBaseline(env_spec=env.spec)
elif v['baseline'] == 'g_mlp':
baseline = GaussianMLPBaseline(env_spec=env.spec)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
discount=v['discount'],
plot=False,
)
# load the state collection from data_upload
load_dir = 'data_upload/state_collections/'
all_feasible_starts = pickle.load(
open(
osp.join(config.PROJECT_PATH, load_dir, 'all_feasible_states.pkl'),
'rb'))
# all_feasible_starts = pickle.load(
# open(osp.join(config.PROJECT_PATH, load_dir, 'key_all_feasible_04_230000.pkl'), 'rb'))
# all_feasible_starts = pickle.load(
# open(osp.join(config.PROJECT_PATH, load_dir, 'key_all_feasible_states_med_rad4.pkl'), 'rb'))
# all_feasible_starts2 = pickle.load(
# open(osp.join(config.PROJECT_PATH, load_dir, 'key_all_feasible_states_min_rad4.pkl'), 'rb'))
# all_feasible_starts3 = pickle.load(
# open(osp.join(config.PROJECT_PATH, load_dir, 'key_all_feasible_states_max_rad2.pkl'), 'rb'))
print("we have %d feasible starts" % all_feasible_starts.size)
all_starts = StateCollection(distance_threshold=v['coll_eps'])
brownian_starts = StateCollection(
distance_threshold=v['regularize_starts'])
logger.log(
'Generating seed starts from the goal (horizon 10, subsample 600 of them)'
)
with algo.env.set_kill_outside(radius=v['kill_radius']):
seed_starts = generate_starts(
env,
starts=[v['start_goal']],
horizon=10, # this is smaller as they are seeds!
variance=v['brownian_variance'],
subsample=v['num_new_starts']) # , animated=True, speedup=10)
# seed_starts = all_feasible_starts.states
# with env.set_kill_outside(radius=0.4):
# find_all_feasible_states(env, seed_starts, distance_threshold=0.1, brownian_variance=1, animate=False)
# # show where these states are:
# shuffled_starts = np.array(all_feasible_starts.state_list)
# np.random.shuffle(shuffled_starts)
# generate_starts(env, starts=shuffled_starts, horizon=100, variance=v['brownian_variance'],
# zero_action=True, animated=True, speedup=10)
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
with algo.env.set_kill_outside(radius=v['kill_radius']):
starts = generate_starts(algo.env,
starts=seed_starts,
horizon=v['brownian_horizon'],
variance=v['brownian_variance'])
# regularization of the brownian starts
brownian_starts.empty()
brownian_starts.append(starts)
starts = brownian_starts.sample(size=v['num_new_starts'])
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
with ExperimentLogger(log_dir,
50 * (outer_iter // 50 + 1),
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment start generator")
algo.env.update_start_generator(
UniformListStateGenerator(
starts.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
# algo.start_worker()
logger.log("Training the algorithm")
algo.current_itr = 0
trpo_paths = algo.train(already_init=outer_iter > 1)
# import pdb; pdb.set_trace()
if v['use_trpo_paths']:
logger.log("labeling starts with trpo rollouts")
[starts, labels] = label_states_from_paths(
trpo_paths,
n_traj=2,
key='goal_reached', # using the min n_traj
as_goal=False,
env=algo.env)
paths = [path for paths in trpo_paths for path in paths]
else:
logger.log("labeling starts manually")
labels, paths = label_states(starts,
algo.env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached',
full_path=True)
with logger.tabular_prefix("OnStarts_"):
algo.env.log_diagnostics(paths)
logger.record_tabular('brownian_starts', brownian_starts.size)
start_classes, text_labels = convert_label(labels)
total_starts = labels.shape[0]
logger.record_tabular('GenStarts_evaluated', total_starts)
start_class_frac = OrderedDict(
) # this needs to be an ordered dict!! (for the log tabular)
for k in text_labels.keys():
frac = np.sum(start_classes == k) / total_starts
logger.record_tabular('GenStart_frac_' + text_labels[k], frac)
start_class_frac[text_labels[k]] = frac
labels = np.logical_and(labels[:, 0],
labels[:, 1]).astype(int).reshape((-1, 1))
logger.log("Labeling on uniform starts")
with logger.tabular_prefix("Uniform_4med_"):
unif_starts = all_feasible_starts.sample(500)
unif_starts = np.pad(unif_starts,
((0, v['start_size'] - unif_starts.shape[1])),
'constant')
mean_reward, paths = evaluate_states(unif_starts,
algo.env,
policy,
v['horizon'],
n_traj=1,
key='goal_reached',
as_goals=False,
full_path=True)
algo.env.log_diagnostics(paths)
# with logger.tabular_prefix("Uniform_4med_bis_"):
# unif_starts = all_feasible_starts.sample(200)
# unif_starts1bis = np.pad(unif_starts, ((0, v['start_size'] - unif_starts.shape[1])), 'constant')
# mean_reward1bis, paths1bis = evaluate_states(unif_starts1bis, algo.env, policy, v['horizon'], n_traj=1,
# key='goal_reached', as_goals=False, full_path=True)
# algo.env.log_diagnostics(paths1bis)
# with logger.tabular_prefix("Uniform_4min_"):
# unif_starts2 = all_feasible_starts2.sample(200)
# unif_starts2 = np.pad(unif_starts2, ((0, v['start_size'] - unif_starts2.shape[1])), 'constant')
# mean_reward2, paths2 = evaluate_states(unif_starts2, algo.env, policy, v['horizon'], n_traj=1,
# key='goal_reached', as_goals=False, full_path=True)
# algo.env.log_diagnostics(paths2)
# with logger.tabular_prefix("Uniform_2max_"):
# unif_starts3 = all_feasible_starts3.sample(200)
# unif_starts3 = np.pad(unif_starts3, ((0, v['start_size'] - unif_starts3.shape[1])), 'constant')
# mean_reward3, paths3 = evaluate_states(unif_starts3, algo.env, policy, v['horizon'], n_traj=1,
# key='goal_reached', as_goals=False, full_path=True)
# algo.env.log_diagnostics(paths3)
logger.dump_tabular(with_prefix=True)
# append new states to list of all starts (replay buffer):
if v['seed_with'] == 'only_goods':
logger.log("Appending good goals to replay and generating seeds")
filtered_raw_starts = [
start for start, label in zip(starts, labels) if label[0] == 1
]
all_starts.append(filtered_raw_starts)
if len(filtered_raw_starts) > 0:
seed_starts = filtered_raw_starts
elif np.sum(start_classes == 0) > np.sum(
start_classes == 1): # if more low reward than high reward
seed_starts = all_starts.sample(
300) # sample them from the replay
else: # add a tone of noise if all the states I had ended up being high_reward!
with algo.env.set_kill_outside(radius=v['kill_radius']):
seed_starts = generate_starts(
algo.env,
starts=starts,
horizon=int(v['horizon'] * 10),
subsample=v['num_new_starts'],
variance=v['brownian_variance'] * 10)
elif v['seed_with'] == 'all_previous':
logger.log("Appending all goals to replay and generating seeds")
all_starts.append(starts)
seed_starts = starts
elif v['seed_with'] == 'on_policy':
all_starts.append(starts)
with algo.env.set_kill_outside(radius=v['kill_radius']):
seed_starts = generate_starts(algo.env,
policy,
horizon=v['horizon'],
subsample=v['num_new_starts'])
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=3)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(AntMazeEnv(maze_id=v['maze_id']))
uniform_goal_generator = UniformStateGenerator(state_size=v['goal_size'],
bounds=v['goal_range'],
center=v['goal_center'])
env = GoalExplorationEnv(
env=inner_env,
goal_generator=uniform_goal_generator,
obs2goal_transform=lambda x: x[-3:-1],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
report.new_row()
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling goals from the GAN")
goals = np.random.uniform(
np.array(v['goal_center']) - np.array(v['goal_range']),
np.array(v['goal_center']) + np.array(v['goal_range']),
size=(300, v['goal_size']))
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment goal generator")
if v['unif_goals']:
env.update_goal_generator(
UniformListStateGenerator(
goals.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
else:
env.update_goal_generator(FixedStateGenerator(v['final_goal']))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
plot=False,
)
algo.train()
logger.log('Generating the Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
logger.log("Labeling the goals")
labels = label_states(goals,
env,
policy,
v['horizon'],
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(goals,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
logger.dump_tabular(with_prefix=False)
report.new_row()
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=3)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
tf_session = tf.Session()
inner_env = normalize(AntMazeEnv(maze_id=v['maze_id']))
uniform_goal_generator = UniformStateGenerator(state_size=v['goal_size'],
bounds=v['goal_range'],
center=v['goal_center'])
env = GoalExplorationEnv(
env=inner_env,
goal_generator=uniform_goal_generator,
obs2goal_transform=lambda x: x[-3:-1],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
report.new_row()
# GAN
logger.log("Instantiating the GAN...")
gan_configs = {key[4:]: value for key, value in v.items() if 'GAN_' in key}
for key, value in gan_configs.items():
if value is tf.train.AdamOptimizer:
gan_configs[key] = tf.train.AdamOptimizer(gan_configs[key +
'_stepSize'])
if value is tflearn.initializations.truncated_normal:
gan_configs[key] = tflearn.initializations.truncated_normal(
stddev=gan_configs[key + '_stddev'])
gan = StateGAN(
state_size=v['goal_size'],
evaluater_size=v['num_labels'],
state_range=v['goal_range'],
state_center=v['goal_center'],
state_noise_level=v['goal_noise_level'],
generator_layers=v['gan_generator_layers'],
discriminator_layers=v['gan_discriminator_layers'],
noise_size=v['gan_noise_size'],
tf_session=tf_session,
configs=gan_configs,
)
all_goals = StateCollection(distance_threshold=v['coll_eps'])
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
feasible_goals = generate_initial_goals(env,
policy,
v['goal_range'],
goal_center=v['goal_center'],
horizon=v['horizon'])
labels = np.ones((feasible_goals.shape[0],
2)).astype(np.float32) # make them all good goals
plot_labeled_states(feasible_goals,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'],
summary_string_base='On-policy Goals:\n')
if v['only_on_policy']:
goals = feasible_goals[np.random.choice(
feasible_goals.shape[0], v['num_new_goals'], replace=False), :]
else:
logger.log("Training the GAN")
gan.pretrain(feasible_goals, v['gan_outer_iters'])
# Sample GAN
logger.log("Sampling goals from the GAN")
raw_goals, _ = gan.sample_states_with_noise(v['num_new_goals'])
if v['replay_buffer'] and outer_iter > 0 and all_goals.size > 0:
old_goals = all_goals.sample(v['num_old_goals'])
goals = np.vstack([raw_goals, old_goals])
else:
goals = raw_goals
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment goal generator")
env.update_goal_generator(
UniformListStateGenerator(
goals.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=0.01,
plot=False,
)
algo.train()
logger.log("Labeling the goals")
labels = label_states(goals,
env,
policy,
v['horizon'],
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(goals,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'])
logger.log('Generating the Heatmap...')
test_and_plot_policy(policy,
env,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new goals to list of all goals (replay buffer): Not the low reward ones!!
filtered_raw_goals = [
goal for goal, label in zip(goals, labels) if label[0] == 1
]
all_goals.append(filtered_raw_goals)
if v['add_on_policy']:
logger.log("sampling on policy")
feasible_goals = generate_initial_goals(
env,
policy,
v['goal_range'],
goal_center=v['goal_center'],
horizon=v['horizon'])
# downsampled_feasible_goals = feasible_goals[np.random.choice(feasible_goals.shape[0], v['add_on_policy']),:]
all_goals.append(feasible_goals)
def run_task(v):
random.seed(v['seed'])
np.random.seed(v['seed'])
sampling_res = 2 if 'sampling_res' not in v.keys() else v['sampling_res']
samples_per_cell = 10 # for the oracle rejection sampling
# Log performance of randomly initialized policy with FIXED goal [0.1, 0.1]
logger.log("Initializing report and plot_policy_reward...")
log_dir = logger.get_snapshot_dir() # problem with logger module here!!
if log_dir is None:
log_dir = "/home/davheld/repos/rllab_goal_rl/data/local/debug"
report = HTMLReport(osp.join(log_dir, 'report.html'), images_per_row=5)
report.add_header("{}".format(EXPERIMENT_TYPE))
report.add_text(format_dict(v))
inner_env = normalize(PointMazeEnv(maze_id=v['maze_id']))
fixed_goal_generator = FixedStateGenerator(state=v['ultimate_goal'])
uniform_start_generator = UniformStateGenerator(state_size=v['start_size'],
bounds=v['start_range'],
center=v['start_center'])
env = GoalStartExplorationEnv(
env=inner_env,
start_generator=uniform_start_generator,
obs2start_transform=lambda x: x[:v['start_size']],
goal_generator=fixed_goal_generator,
obs2goal_transform=lambda x: x[:v['goal_size']],
terminal_eps=v['terminal_eps'],
distance_metric=v['distance_metric'],
extend_dist_rew=v['extend_dist_rew'],
only_feasible=v['only_feasible'],
terminate_env=True,
)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain'],
init_std=v['policy_init_std'],
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# initialize all logging arrays on itr0
outer_iter = 0
logger.log('Generating the Initial Heatmap...')
plot_policy_means(policy,
env,
sampling_res=sampling_res,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
test_and_plot_policy(policy,
env,
as_goals=False,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
center=v['goal_center'],
limit=v['goal_range'])
report.new_row()
all_starts = StateCollection(distance_threshold=v['coll_eps'])
# Use asymmetric self-play to run Alice to generate starts for Bob.
# Use a double horizon because the horizon is shared between Alice and Bob.
env_alice = AliceEnv(env_alice=env,
env_bob=env,
policy_bob=policy,
max_path_length=v['alice_horizon'],
alice_factor=v['alice_factor'],
alice_bonus=v['alice_bonus'],
gamma=1,
stop_threshold=v['stop_threshold'])
policy_alice = GaussianMLPPolicy(
env_spec=env_alice.spec,
hidden_sizes=(64, 64),
# Fix the variance since different goals will require different variances, making this parameter hard to learn.
learn_std=v['learn_std'],
adaptive_std=v['adaptive_std'],
std_hidden_sizes=(16,
16), # this is only used if adaptive_std is true!
output_gain=v['output_gain_alice'],
init_std=v['policy_init_std_alice'],
)
baseline_alice = LinearFeatureBaseline(env_spec=env_alice.spec)
algo_alice = TRPO(
env=env_alice,
policy=policy_alice,
baseline=baseline_alice,
batch_size=v['pg_batch_size_alice'],
max_path_length=v['alice_horizon'],
n_itr=v['inner_iters_alice'],
step_size=0.01,
discount=v['discount_alice'],
plot=False,
)
for outer_iter in range(1, v['outer_iters']):
logger.log("Outer itr # %i" % outer_iter)
logger.log("Sampling starts")
starts, t_alices = generate_starts_alice(
env_alice=env_alice,
algo_alice=algo_alice,
start_states=[v['start_goal']],
num_new_starts=v['num_new_starts'],
log_dir=log_dir)
labels = label_states(starts,
env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(starts,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'],
summary_string_base='initial starts labels:\n')
report.save()
if v['replay_buffer'] and outer_iter > 0 and all_starts.size > 0:
old_starts = all_starts.sample(v['num_old_starts'])
starts = np.vstack([starts, old_starts])
with ExperimentLogger(log_dir,
'last',
snapshot_mode='last',
hold_outter_log=True):
logger.log("Updating the environment start generator")
env.update_start_generator(
UniformListStateGenerator(
starts.tolist(),
persistence=v['persistence'],
with_replacement=v['with_replacement'],
))
logger.log("Training the algorithm")
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=v['pg_batch_size'],
max_path_length=v['horizon'],
n_itr=v['inner_iters'],
step_size=v['step_size'],
discount=v['discount'],
plot=False,
)
# We don't use these labels anyway, so we might as well take them from training.
#trpo_paths = algo.train()
algo.train()
# logger.log("labeling starts with trpo rollouts")
# [starts, labels] = label_states_from_paths(trpo_paths, n_traj=2, key='goal_reached', # using the min n_traj
# as_goal=False, env=env)
# paths = [path for paths in trpo_paths for path in paths]
with logger.tabular_prefix('Outer_'):
logger.record_tabular('t_alices', np.mean(t_alices))
logger.log('Generating the Heatmap...')
plot_policy_means(policy,
env,
sampling_res=sampling_res,
report=report,
limit=v['goal_range'],
center=v['goal_center'])
test_and_plot_policy(policy,
env,
as_goals=False,
max_reward=v['max_reward'],
sampling_res=sampling_res,
n_traj=v['n_traj'],
itr=outer_iter,
report=report,
center=v['goal_center'],
limit=v['goal_range'])
logger.log("Labeling the starts")
labels = label_states(starts,
env,
policy,
v['horizon'],
as_goals=False,
n_traj=v['n_traj'],
key='goal_reached')
plot_labeled_states(starts,
labels,
report=report,
itr=outer_iter,
limit=v['goal_range'],
center=v['goal_center'],
maze_id=v['maze_id'])
# ###### extra for deterministic:
# logger.log("Labeling the goals deterministic")
# with policy.set_std_to_0():
# labels_det = label_states(goals, env, policy, v['horizon'], n_traj=v['n_traj'], n_processes=1)
# plot_labeled_states(goals, labels_det, report=report, itr=outer_iter, limit=v['goal_range'], center=v['goal_center'])
labels = np.logical_and(labels[:, 0],
labels[:, 1]).astype(int).reshape((-1, 1))
logger.dump_tabular(with_prefix=False)
report.new_row()
# append new states to list of all starts (replay buffer): Not the low reward ones!!
filtered_raw_starts = [
start for start, label in zip(starts, labels) if label[0] == 1
]
if len(
filtered_raw_starts
) == 0: # add a tone of noise if all the states I had ended up being high_reward!
logger.log("Bad Alice! All goals are high reward!")
# seed_starts = filtered_raw_starts
# else:
# seed_starts = generate_starts(env, starts=starts, horizon=v['horizon'] * 2, subsample=v['num_new_starts'],
# variance=v['brownian_variance'] * 10)
all_starts.append(filtered_raw_starts)
