# https://iopscience.iop.org/article/10.1088/0143-0807/37/6/065008/pdf
# Importantly they depict the threshold
# for admissible acceleration onset or jerk as j = 15g/s or ~150m/s^3.
jerk_threshold=150.0, # 15g/s
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
physics_steps_per_observation=12,
discrete_actions=COMFORTABLE_ACTIONS2,
)
net_config = dict(hidden_units=(256, 256, 256), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add(
'resume',
'/home/c2/src/tmp/spinningup/data/intersection_discrete_micro_turn_lower_lane_pen2_diag_lane20/intersection_discrete_micro_turn_lower_lane_pen2_diag_lane20_s0_2020_05-16_12-44.32.941234'
)
# eg.add('reinitialize_optimizer_on_resume', True)
# eg.add('num_inputs_to_add', 0)
eg.add('pi_lr', 3e-4) # default pi_lr=3e-4
eg.add('vf_lr', 1e-3) # default vf_lr=1e-3,
# eg.add('boost_explore', 5)
eg.add('epochs', 20000)
eg.add('steps_per_epoch', 8000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
import torch
from gym_match_input_continuous.experiments import utils
experiment_name = os.path.basename(__file__)[:-3]
notes = """
Trying to shift advantages by one standard deviation
vs 90: https://photos.app.goo.gl/MZ9p9sdjFBf3yZba8
"""
env_config = dict(env_name='match-input-continuous-v0', )
net_config = dict(hidden_units=(32, 32), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('gamma', 0.999) # Lower gamma so seconds of effective horizon remains at 10s with current physics steps = 12 * 1/60s * 1 / (1-gamma)
eg.add('epochs', 1000)
eg.add('steps_per_epoch', 500)
eg.add('shift_advs_pct', 68)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
def train():
eg.run(ppo_pytorch)
def run_vpg_lava():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=3)
args = parser.parse_args()
eg = ExperimentGrid(name='vpg-pt-bench')
eg.add('env_name', 'MiniGrid-LavaCrossingS9N2-v1', '', True)
eg.add('seed', [6 * i for i in range(args.num_runs)])
eg.add('epochs', 500)
eg.add('steps_per_epoch', 5000)
eg.add('max_ep_len', 200)
# eg.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
eg.add('ac_kwargs:hidden_sizes', [(32, )], 'hid')
# eg.add('ac_kwargs:activation', [torch.nn.Tanh, torch.nn.ReLU], '')
eg.add('ac_kwargs:activation', [torch.nn.Tanh], '')
eg.run(vpg_pytorch, num_cpu=args.cpu, datestamp=True)
gforce_penalty_coeff=0.06,
collision_penalty_coeff=4,
lane_penalty_coeff=0.02,
speed_reward_coeff=0.50,
end_on_harmful_gs=False,
end_on_lane_violation=False,
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
physics_steps_per_observation=12,
)
net_config = dict(hidden_units=(256, 256), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add(
'resume',
'/workspace/dd0-data-resume1/intersection_2_agents_fine_tune_add_left_yield_from_scratch/intersection_2_agents_fine_tune_add_left_yield_from_scratch_s0_2020_03-25_21-16.12/best_EpRet/2020_03-27_11-01.22'
)
# eg.add('reinitialize_optimizer_on_resume', True)
# eg.add('num_inputs_to_add', 0)
eg.add('pi_lr', 3e-6)
eg.add('vf_lr', 1e-5)
# eg.add('boost_explore', 5)
eg.add('epochs', 8000)
eg.add('steps_per_epoch', 32000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
expect_normalized_action_deltas=False,
jerk_penalty_coeff=0.10 / (60*10), # dropped from 0.10, see notes
gforce_penalty_coeff=0.031,
collision_penalty_coeff=1,
gforce_threshold=None,
incent_win=True,
constrain_controls=False,
)
net_config = dict(
hidden_units=(256, 256),
activation=torch.nn.Tanh
)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add('resume', '/home/c2/src/tmp/spinningup/data/intersection_2_agents_fine_tune_collision_resume_add_comfort/intersection_2_agents_fine_tune_collision_resume_add_comfort_s0_2020_03-13_15-24.07/best_trip_pct/2020_03-13_16-18.59')
eg.add('reinitialize_optimizer_on_resume', True)
eg.add('pi_lr', 3e-6) # doesn't seem to have an effect, but playing it safe and lowering learning rate since we're not restoring adam rates
eg.add('vf_lr', 1e-5) # doesn't seem to have an effect, but playing it safe and lowering learning rate since we're not restoring adam rates
eg.add('epochs', 8000)
eg.add('steps_per_epoch', 8000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
def train():
eg.run(ppo_pytorch)
def parse_and_execute_grid_search(cmd, args):
"""Interprets algorithm name and cmd line args into an ExperimentGrid."""
# Parse which algorithm to execute
algo = eval('spinup.' + cmd)
# Before all else, check to see if any of the flags is 'help'.
valid_help = ['--help', '-h', 'help']
if any([arg in valid_help for arg in args]):
print('\n\nShowing docstring for spinup.' + cmd + ':\n')
print(algo.__doc__)
sys.exit()
def process(arg):
# Process an arg by eval-ing it, so users can specify more
# than just strings at the command line (eg allows for
# users to give functions as args).
try:
return eval(arg)
except:
return arg
# Make first pass through args to build base arg_dict. Anything
# with a '--' in front of it is an argument flag and everything after,
# until the next flag, is a possible value.
arg_dict = dict()
for i, arg in enumerate(args):
assert i > 0 or '--' in arg, \
friendly_err("You didn't specify a first flag.")
if '--' in arg:
arg_key = arg.lstrip('-')
arg_dict[arg_key] = []
else:
arg_dict[arg_key].append(process(arg))
# Make second pass through, to catch flags that have no vals.
# Assume such flags indicate that a boolean parameter should have
# value True.
for k, v in arg_dict.items():
if len(v) == 0:
v.append(True)
# Third pass: check for user-supplied shorthands, where a key has
# the form --keyname[kn]. The thing in brackets, 'kn', is the
# shorthand. NOTE: modifying a dict while looping through its
# contents is dangerous, and breaks in 3.6+. We loop over a fixed list
# of keys to avoid this issue.
given_shorthands = dict()
fixed_keys = list(arg_dict.keys())
for k in fixed_keys:
p1, p2 = k.find('['), k.find(']')
if p1 >= 0 and p2 >= 0:
# Both '[' and ']' found, so shorthand has been given
k_new = k[:p1]
shorthand = k[p1 + 1:p2]
given_shorthands[k_new] = shorthand
arg_dict[k_new] = arg_dict[k]
del arg_dict[k]
# Penultimate pass: sugar. Allow some special shortcuts in arg naming,
# eg treat "env" the same as "env_name". This is super specific
# to Spinning Up implementations, and may be hard to maintain.
# These special shortcuts are described by SUBSTITUTIONS.
for special_name, true_name in SUBSTITUTIONS.items():
if special_name in arg_dict:
# swap it in arg dict
arg_dict[true_name] = arg_dict[special_name]
del arg_dict[special_name]
if special_name in given_shorthands:
# point the shortcut to the right name
given_shorthands[true_name] = given_shorthands[special_name]
del given_shorthands[special_name]
# Final pass: check for the special args that go to the 'run' command
# for an experiment grid, separate them from the arg dict, and make sure
# that they have unique values. The special args are given by RUN_KEYS.
run_kwargs = dict()
for k in RUN_KEYS:
if k in arg_dict:
val = arg_dict[k]
assert len(val)==1, \
friendly_err("You can only provide one value for %s."%k)
run_kwargs[k] = val[0]
del arg_dict[k]
# Determine experiment name. If not given by user, will be determined
# by the algorithm name.
if 'exp_name' in arg_dict:
assert len(arg_dict['exp_name'])==1, \
friendly_err("You can only provide one value for exp_name.")
exp_name = arg_dict['exp_name'][0]
del arg_dict['exp_name']
else:
exp_name = 'cmd_' + cmd
# Make sure that if num_cpu > 1, the algorithm being used is compatible
# with MPI.
if 'num_cpu' in run_kwargs and not (run_kwargs['num_cpu'] == 1):
assert cmd in MPI_COMPATIBLE_ALGOS, \
friendly_err("This algorithm can't be run with num_cpu > 1.")
# Special handling for environment: make sure that env_name is a real,
# registered gym environment.
valid_envs = [e.id for e in list(gym.envs.registry.all())]
assert 'env_name' in arg_dict, \
friendly_err("You did not give a value for --env_name! Add one and try again.")
for env_name in arg_dict['env_name']:
err_msg = dedent("""
%s is not registered with Gym.
Recommendations:
* Check for a typo (did you include the version tag?)
* View the complete list of valid Gym environments at
https://gym.openai.com/envs/
""" % env_name)
assert env_name in valid_envs, err_msg
# Construct and execute the experiment grid.
eg = ExperimentGrid(name=exp_name)
for k, v in arg_dict.items():
eg.add(k, v, shorthand=given_shorthands.get(k))
eg.run(algo, **run_kwargs)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--env', type=str, default='HalfCheetah-v2')
parser.add_argument('--h', type=int, default=300)
parser.add_argument('--l', type=int, default=1)
parser.add_argument('--num_runs', '-n', type=int, default=3)
parser.add_argument('--steps_per_epoch', '-s', type=int, default=5000)
parser.add_argument('--total_steps', '-t', type=int, default=int(5e4))
args = parser.parse_args()
def td3_with_actor_critic(**kwargs):
td3(ac_kwargs=dict(hidden_sizes=[args.h] * args.l),
start_steps=5000,
max_ep_len=150,
batch_size=64,
polyak=0.95,
**kwargs)
eg = ExperimentGrid(name='ex2-3_td3')
eg.add('replay_size', int(args.total_steps))
eg.add('env_name', args.env, '', True)
eg.add('seed', [10 * i for i in range(args.num_runs)])
eg.add('epochs', int(args.total_steps / args.steps_per_epoch))
eg.add('steps_per_epoch', args.steps_per_epoch)
eg.add('remove_action_clip', [False, True])
eg.run(td3_with_actor_critic, datestamp=True)
from spinup import sac_pytorch
import torch
import gym
TASK_HORIZON = 200
NUM_TASKS = 10
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=8)
parser.add_argument('--num_runs', type=int, default=1)
args = parser.parse_args()
eg = ExperimentGrid(name='sac-MT10')
eg.add('env_name', 'MT10Helper-v0', '', True)
eg.add('num_tasks', [10])
eg.add('seed', [10 * i for i in range(args.num_runs)])
eg.add('epochs', 1000)
eg.add('steps_per_epoch', TASK_HORIZON * NUM_TASKS)
eg.add('lr', [3e-4])
eg.add('start_steps', 1000)
eg.add('ac_kwargs:hidden_sizes', [(400, 400)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.ReLU], '')
eg.run(sac_pytorch, num_cpu=args.cpu)
#from metaworld.benchmarks import MT10
#
#env_fn = lambda : MTEnv(MT10.get_train_tasks())
#
#ac_kwargs = dict(hidden_sizes=[400,400], activation=torch.nn.ReLU)
#
from spinup.utils.run_utils import ExperimentGrid
from spinup import dvpg_pytorch
import torch
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=2)
parser.add_argument('--num_runs', type=int, default=10)
args = parser.parse_args()
eg = ExperimentGrid(name='dvpg_LunarLander-v2')
eg.add('env_name', 'LunarLander-v2', '', True)
eg.add('seed', [10 * i for i in range(args.num_runs)])
eg.add('epochs', 500)
eg.add('epsilon', [0.01, 0.1, 1, 10])
eg.add('pi_lr', [3e-4, 3e-3, 3e-2])
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [(32, 32)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.Tanh], '')
eg.run(dvpg_pytorch, num_cpu=args.cpu)
def runExperiment(someEnv,someOptimizer, someActivation):
env = ExperimentGrid(name='vpg-trece')
env.add('env_name', someEnv, '', True)
# eg.add('clip_ratio', [0.1,0.2])
env.add('seed', [10*i for i in range(args.num_runs)])
env.add('epochs', 10)
env.add('steps_per_epoch', [4000])
#someOptimizer should be list
env.add('optimizer',[someOptimizer])
env.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
env.add('ac_kwargs:activation', [someActivation], '')
env.run(vpg, num_cpu=args.cpu)
jerk_penalty_coeff=0.20 / (60*100),
gforce_penalty_coeff=0.06,
collision_penalty_coeff=4,
end_on_harmful_gs=False,
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
)
net_config = dict(
hidden_units=(256, 256),
activation=torch.nn.Tanh
)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add('resume', '/home/c2/src/tmp/spinningup/data/intersection_2_agents_fine_tune_add_left_yield/intersection_2_agents_fine_tune_add_left_yield_s0_2020_03-23_13-16.15')
eg.add('reinitialize_optimizer_on_resume', False) # Old optimizer had only 28 inputs despite NN having 29!
eg.add('num_inputs_to_add', 0)
eg.add('pi_lr', 3e-6)
eg.add('vf_lr', 1e-5)
eg.add('boost_explore', 5)
eg.add('epochs', 8000)
eg.add('steps_per_epoch', 32000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
help=
'if set to true, then output out the material type first, then condition the material thickness on the material type'
)
parser.add_argument('--use_rnn', action='store_true')
parser.add_argument('--spectrum_repr', action='store_true')
args = parser.parse_args()
env_kwargs = {
"discrete_thick": args.discrete_thick,
'spectrum_repr': args.spectrum_repr,
'bottom_up': False,
'merit_func': cal_reward
}
eg = ExperimentGrid(name=args.exp_name)
eg.add('env_fn', get_env_fn(args.env, **env_kwargs))
eg.add('seed', [42 * (i + 1) for i in range(args.num_runs)])
eg.add('epochs', 3000)
eg.add('steps_per_epoch', 1000)
eg.add('ac_kwargs:hidden_sizes', [(64, )], 'hid')
eg.add('ac_kwargs:cell_size', 64, '')
eg.add('ac_kwargs:not_repeat', [True, False])
eg.add('ac_kwargs:ortho_init', ['on'])
eg.add('ac_kwargs:hierarchical', [True, False])
eg.add('ac_kwargs:channels', 16)
eg.add('ac_kwargs:act_emb', [True])
eg.add('ac_kwargs:act_emb_dim', 5)
eg.add('use_rnn', args.use_rnn)
eg.add('gamma', 1)
eg.add('beta', [0.01])
eg.add('lam', [0.95])
from spinup.utils.run_utils import ExperimentGrid
from spinup import vpg_pytorch
import torch
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=8)
parser.add_argument('--num_runs', type=int, default=5)
args = parser.parse_args()
eg = ExperimentGrid(name='vpg_LunarLander-v2')
eg.add('env_name', 'LunarLander-v2', '', True)
eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 500)
eg.add('pi_lr', [3e-4, 3e-3, 3e-2])
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [(32,32)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.Tanh], '')
eg.run(vpg_pytorch, num_cpu=args.cpu)
from spinup.utils.run_utils import ExperimentGrid
from spinup import sac_pytorch
import torch
import gym
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=1)
args = parser.parse_args()
eg = ExperimentGrid(name='sac-goal')
eg.add('env_name', 'SawyerPush-v0', '', True)
eg.add('seed', [10 * i for i in range(args.num_runs)])
eg.add('epochs', 300)
eg.add('steps_per_epoch', 10 * 150)
eg.add('lr', [3e-4, 1e-3])
eg.add('start_steps', 1000)
eg.add('ac_kwargs:hidden_sizes', [(400, 400)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.ReLU], '')
eg.run(sac_pytorch, num_cpu=args.cpu)
#from metaworld.benchmarks import MT10
#
#env_fn = lambda : MTEnv(MT10.get_train_tasks())
#
#ac_kwargs = dict(hidden_sizes=[400,400], activation=torch.nn.ReLU)
#
#logger_kwargs = dict(output_dir='~/spinup/data/', exp_name='SAC_MT10')
#
#sac_pytorch(env_fn=env_fn, ac_kwargs=ac_kwargs, steps_per_epoch=128 * 10, epochs=1000, start_steps=1000, lr=3e-4, logger_kwargs=logger_kwargs)
# https://iopscience.iop.org/article/10.1088/0143-0807/37/6/065008/pdf
# Importantly they depict the threshold
# for admissible acceleration onset or jerk as j = 15g/s or ~150m/s^3.
jerk_threshold=150.0, # 15g/s
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
physics_steps_per_observation=12,
discrete_actions=COMFORTABLE_ACTIONS2,
)
net_config = dict(hidden_units=(256, 256), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add(
'resume',
'/home/c2/src/tmp/spinningup/data/intersection_discrete_micro_turn_lower_lane_pen2_diag_lane14_2/intersection_discrete_micro_turn_lower_lane_pen2_diag_lane14_2_s0_2020_05-12_18-26.22.747749'
)
# eg.add('reinitialize_optimizer_on_resume', True)
# eg.add('num_inputs_to_add', 0)
# eg.add('pi_lr', 3e-6)
# eg.add('vf_lr', 1e-5)
# eg.add('boost_explore', 5)
eg.add('epochs', 20000)
eg.add('steps_per_epoch', 8000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
# for admissible acceleration onset or jerk as j = 15g/s or ~150m/s^3.
jerk_threshold=150.0, # 15g/s
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
physics_steps_per_observation=12,
discrete_actions=COMFORTABLE_ACTIONS,
)
net_config = dict(
hidden_units=(256, 256),
activation=torch.nn.Tanh
)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add('resume', '/home/c2/src/tmp/spinningup/data/intersection_from_scratch_discrete_steer_and_accel_resume_snapshot/intersection_from_scratch_discrete_steer_and_accel_resume_s0_2020_04-22_13-55.32.354696')
# eg.add('reinitialize_optimizer_on_resume', True)
# eg.add('num_inputs_to_add', 0)
# eg.add('pi_lr', 3e-6)
# eg.add('vf_lr', 1e-5)
# eg.add('boost_explore', 5)
eg.add('epochs', 20000)
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
remainder = remainder % division
indexes.append(index)
total = division
actual_setting = {}
for j in range(len(indexes)):
actual_setting[setting_names[j]] = settings[j][indexes[j]]
return indexes, actual_setting
indexes, actual_setting = get_setting(args.setting, total, settings,
setting_names)
eg = ExperimentGrid(name=EXPERIMENT_NAME)
# use eg.add to add parameters in the settings or add parameters tha apply to all jobs
# we now automated this part, as long as you added settings correctly into the arrays at the start of this program
# they should be added to experiment automatically
for i in range(len(actual_setting)):
setting_name = setting_names[i]
if setting_name != 'env_name' and setting_name != 'seed':
eg.add(setting_name, actual_setting[setting_name],
setting_savename_prefix[i], whether_add_to_savename[i])
eg.add('env_name', actual_setting['env_name'], '', True)
eg.add('seed', actual_setting['seed'])
eg.run(function_to_run, num_cpu=args.cpu, data_dir=save_data_dir)
print(
'\n###################################### GRID EXP END ######################################'
)
print('total time for grid experiment:', time.time() - start_time)
from spinup.utils.run_utils import ExperimentGrid
from spinup import ppo
from spinup import vpg
import tensorflow as tf
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=3)
args = parser.parse_args()
eg = ExperimentGrid(name='vpg-bench')
eg.add('env_name', 'Pong-ram-v0', '', True)
#eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 10000)
#eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [200], 'hid')
eg.add('ac_kwargs:activation', [tf.nn.relu], '')
eg.run(vpg, num_cpu=args.cpu)
from spinup.utils.run_utils import ExperimentGrid
from spinup import ppo
import tensorflow as tf
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=3)
args = parser.parse_args()
eg = ExperimentGrid(name='ppo-bench')
eg.add('env_name', 'CartPole-v0', '', True)
eg.add('seed', [10 * i for i in range(args.num_runs)])
eg.add('epochs', 10)
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [(32, ), (64, 64)], 'hid')
eg.add('ac_kwargs:activation', [tf.tanh, tf.nn.relu], '')
eg.run(ppo, num_cpu=args.cpu)
def train():
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add(
'resume',
'/home/c2/src/tmp/spinningup/data/deepdrive-2d-intersection-no-constrained-controls-example/deepdrive-2d-intersection-no-constrained-controls-example_s0_2020_03-10_13-14.50/best_HorizonReturn/2020_03-11_11-36.27'
)
eg.add('reinitialize_optimizer_on_resume', False)
eg.add(
'pi_lr', 3e-5
) # doesn't seem to have an effect, but playing it safe and lowering learning rate since we're not restoring adam rates
eg.add(
'vf_lr', 1e-4
) # doesn't seem to have an effect, but playing it safe and lowering learning rate since we're not restoring adam rates
eg.add('epochs', 8000)
# eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', (256, 256), 'hid')
eg.add('ac_kwargs:activation', torch.nn.Tanh, '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
eg.run(ppo_pytorch)
# Use default hidden sizes in actor_critic function, comment below out
eg.add('ac_kwargs:hidden_sizes', [(16,16)], 'hid')
eg.add('ac_kwargs:activation', [tf.nn.relu], '')
eg.run(algo[i], num_cpu=args.cpu)
'''
#Training
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=1)
parser.add_argument('--num_runs', type=int, default=1)
args = parser.parse_args()
eg = ExperimentGrid(name=ex_number)
eg.add('env_name', env_name, '', True)
eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 5)
#eg.add('steps_per_epoch', 4000)
#eg.add('max_ep_len', 1500)
eg.add('saveCheckpointEpochStep', 1)
eg.add('ac_kwargs:activation', [tf.nn.relu], '')
eg.add('ac_kwargs:hidden_sizes', [(8,8)], 'hid')
eg.run(ddpg, num_cpu=args.cpu)
from spinup.utils.run_utils import ExperimentGrid
from spinup import ppo_pytorch, vpg_pytorch
import torch
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=3)
parser.add_argument('--algo', type=str, default='vpg',
choices=['vpg', 'ppo'])
args = parser.parse_args()
eg = ExperimentGrid(name=args.algo+'-pyt-bench')
eg.add('env_name', 'sorty:sorty-v0', 'n=6', True)
eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 10)
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.Tanh, torch.nn.ReLU], '')
if args.algo == 'vpg':
eg.run(vpg_pytorch, num_cpu=args.cpu)
elif args.algo == 'ppo':
eg.run(ppo_pytorch, num_cpu=args.cpu)
else:
raise ValueError("Unsupported Training Algorithm!")
from spinup import vpg
import tensorflow as tf
# ENVS
# BipedalWalkerHardcore-v2
# LunarLanderContinuous-v2
# MontezumaRevenge-ram-v0
# Enduro-ram-v0
# MsPacman-ram-v0
# Ant-v2
# HumanoidStandup-v2
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=1)
parser.add_argument('--num_runs', type=int, default=10)
# parser.add_argument('--clip_ratio', type=int, )
args = parser.parse_args()
Enduro = ExperimentGrid(name='vpg-nueve-singular')
Enduro.add('env_name', 'Enduro-ram-v0', '', True)
# eg.add('clip_ratio', [0.1,0.2])
Enduro.add('seed', [10*i for i in range(args.num_runs)])
Enduro.add('epochs', 10)
Enduro.add('steps_per_epoch', [4000,100])
Enduro.add('optimizer',['GradientDescentOptimizer', 'MomentumOptimizer', 'ProximalAdagradOptimizer', 'ProximalGradientDescentOptimizer', 'RMSPropOptimizer', 'AdaMaxOptimizer', 'AdamGSOptimizer', 'AdamWOptimizer', 'AddSignOptimizer', 'GGTOptimizer', 'LARSOptimizer', 'LazyAdamGSOptimizer', 'LazyAdamOptimizer', 'MomentumWOptimizer', 'NadamOptimizer', 'PowerSignOptimizer', 'RegAdagradOptimizer', 'ShampooOptimizer'])
Enduro.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
Enduro.add('ac_kwargs:activation', [tf.nn.tanh], '')
Enduro.run(vpg, num_cpu=args.cpu)
#, 'ProximalAdagradOptimizer', 'ProximalGradientDescentOptimizer', 'RMSPropOptimizer', 'AdaMaxOptimizer', 'AdamGSOptimizer', 'AdamWOptimizer', 'AddSignOptimizer', 'GGTOptimizer', 'LARSOptimizer', 'LazyAdamGSOptimizer', 'LazyAdamOptimizer', 'MomentumWOptimizer', 'NadamOptimizer', 'PowerSignOptimizer', 'RegAdagradOptimizer', 'ShampooOptimizer'
# , , , , tf.nn.selu, tf.nn.softplus, tf.nn.softsign, tf.sigmoid, tf.tanh
from spinup import vpg
import tensorflow as tf
# ENVS
# BipedalWalkerHardcore-v2
# LunarLanderContinuous-v2
# MontezumaRevenge-ram-v0
# Enduro-ram-v0
# MsPacman-ram-v0
# Ant-v2
# HumanoidStandup-v2
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--cpu', type=int, default=4)
parser.add_argument('--num_runs', type=int, default=10)
# parser.add_argument('--clip_ratio', type=int, )
args = parser.parse_args()
HumanoidStandup = ExperimentGrid(name='vpg-ocho')
HumanoidStandup.add('env_name', 'HumanoidStandup-v2', '', True)
# eg.add('clip_ratio', [0.1,0.2])
HumanoidStandup.add('seed', [10*i for i in range(args.num_runs)])
HumanoidStandup.add('epochs', 10)
HumanoidStandup.add('steps_per_epoch', [4000,100])
HumanoidStandup.add('optimizer',['LARSOptimizer', 'LazyAdamGSOptimizer', 'LazyAdamOptimizer', 'MomentumWOptimizer', 'NadamOptimizer', 'PowerSignOptimizer', 'RegAdagradOptimizer', 'ShampooOptimizer'])
HumanoidStandup.add('ac_kwargs:hidden_sizes', [(300,), (128,128)], 'hid')
HumanoidStandup.add('ac_kwargs:activation', [tf.nn.relu, tf.nn.relu6, tf.nn.crelu, tf.nn.elu, tf.nn.selu, tf.nn.softplus, tf.nn.softsign, tf.sigmoid, tf.tanh], '')
HumanoidStandup.run(vpg, num_cpu=args.cpu)
def train():
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add('epochs', 8000)
eg.add('gamma', 0.95)
eg.add('lam', 0.835)
# eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', (256, 256), 'hid')
eg.add('ac_kwargs:activation', torch.nn.Tanh, '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
eg.run(ppo_pytorch)
def run_experiment(args):
def env_fn():
import offload_env # registers custom envs to gym env registry
env = gym.make('offload-v0')
return env
eg = ExperimentGrid(name=args.exp_name)
eg.add('env_fn', env_fn)
eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 10)
eg.add('steps_per_epoch', 10000)
eg.add('save_freq', 20)
eg.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
eg.add('ac_kwargs:activation', [tf.tanh, tf.nn.relu], '')
eg.run(vpg, num_cpu=args.cpu)
env_name='deepdrive-2d-intersection-w-gs-allow-decel-v0',
is_intersection_map=True,
expect_normalized_action_deltas=False,
jerk_penalty_coeff=0.20 / (60 * 100),
gforce_penalty_coeff=0.06,
collision_penalty_coeff=4,
gforce_threshold=None,
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
)
net_config = dict(hidden_units=(256, 256), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
# eg.add('seed', 0)
eg.add(
'resume',
'/home/c2/src/tmp/spinningup/data/intersection_2_agents_fine_tune_add_left_yield2/intersection_2_agents_fine_tune_add_left_yield2_s0_2020_03-23_22-40.11'
)
eg.add('reinitialize_optimizer_on_resume', True)
eg.add('num_inputs_to_add', 0)
eg.add('pi_lr', 3e-6)
eg.add('vf_lr', 1e-5)
# eg.add('boost_explore', 5)
eg.add('epochs', 8000)
eg.add('steps_per_epoch', 32000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
def run_experiment(args):
# def env_fn():
# import flexibility # register flexibility to gym env registry
# return gym.make(args.env_name)
eg = ExperimentGrid(name=args.exp_name)
eg.add('seed', [10*i for i in range(args.num_runs)] if args.seed is None else args.seed)
eg.add('epochs', args.epochs)
eg.add('steps_per_epoch', args.steps_per_epoch)
eg.add('save_freq', args.save_freq)
eg.add('max_ep_len', 200)
eg.add('ac_kwargs:activation', eval(args.act), '')
eg.add('custom_h', args.custom_h)
eg.add('do_checkpoint_eval', args.do_checkpoint_eval)
eg.add('eval_episodes', args.eval_episodes)
eg.add('train_v_iters', args.train_v_iters)
eg.add('eval_temp', args.eval_temp)
eg.add('train_starting_temp', args.train_starting_temp)
eg.add('gamma', args.gamma)
eg.add('env_version', args.env_version)
eg.add('env_name', args.env_name)
eg.add('env_subtract_full_flex', args.env_subtract_full_flex)
eg.add('meta_learning', args.meta_learning)
eg.add('finetune', args.finetune)
eg.add('finetune_model_path', args.finetune_model_path)
eg.add('lam', args.lam)
eg.add('early_stop_epochs', args.early_stop_epochs)
eg.add('save_all_eval', args.save_all_eval)
if args.episodes_per_epoch is not None:
eg.add('episodes_per_epoch', args.episodes_per_epoch)
if args.env_version >= 3:
# args.file_path = "/home/user/git/spinningup/spinup/FlexibilityEnv/input_m8n12_cv0.8.pkl"
prefix = os.getcwd().split('RL_flex_design')[0]
args.file_path = prefix + "RL_flex_design/spinup/FlexibilityEnv_input/{}".format(args.env_input)
m, n, mean_c, mean_d, sd_d, profit_mat, target_arcs, fixed_costs, flex_0 = load_FlexibilityEnv_input(args.file_path)
eg.add('env_input', args.file_path)
eg.add('env_n_sample', args.env_n_sample)
if args.target_arcs is None:
eg.add('target_arcs', target_arcs)
else: # target_arcs is explicitly specified by the scripts, which overrides the target_arc from the input file
eg.add('target_arcs', args.target_arcs)
if args.algo == "ppo":
eg.add('train_pi_iters', args.train_pi_iters)
eg.run(ppo)
elif args.algo == "vpg":
eg.run(vpg)
jerk_penalty_coeff=3.3e-5,
gforce_penalty_coeff=0.006 * 5,
collision_penalty_coeff=4,
lane_penalty_coeff=0.02,
speed_reward_coeff=0.50,
gforce_threshold=None,
incent_win=True,
constrain_controls=False,
incent_yield_to_oncoming_traffic=True,
physics_steps_per_observation=12,
)
net_config = dict(hidden_units=(64, 64), activation=torch.nn.Tanh)
eg = ExperimentGrid(name=experiment_name)
eg.add('env_name', env_config['env_name'], '', False)
pso = env_config['physics_steps_per_observation']
effective_horizon_seconds = 10
eg.add(
'gamma', 1 - pso / (effective_horizon_seconds * FPS)
) # Lower gamma so seconds of effective horizon remains at 10s with current physics steps = 12 * 1/60s * 1 / (1-gamma)
eg.add('epochs', 417)
eg.add('try_rollouts', 2)
eg.add('steps_per_try_rollout', 1)
eg.add('take_worst_rollout', True)
eg.add('steps_per_epoch', 20000)
eg.add('ac_kwargs:hidden_sizes', net_config['hidden_units'], 'hid')
eg.add('ac_kwargs:activation', net_config['activation'], '')
eg.add('notes', notes, '')
eg.add('run_filename', os.path.realpath(__file__), '')
eg.add('env_config', env_config, '')
if __name__ == '__main__':
import argparse, yaml
from dotmap import DotMap
parser = argparse.ArgumentParser()
parser.add_argument('params_file', type=str, default=None)
parser.add_argument('--cloud', default=False, action="store_true")
args = parser.parse_args()
assert args.params_file is not None
params = None
with open(args.params_file) as f:
params = DotMap(yaml.load(f))
eg = ExperimentGrid(name=params.name)
eg.add('env_name', params.env_name, '', True)
eg.add('test_env_names', params.test_env_names, "test", False)
eg.add('seed', params.seed)
eg.add('epochs', params.epochs)
eg.add('steps_per_epoch', params.steps_per_epoch)
eg.add('update_after', params.update_after)
eg.add('alpha', params.alpha, 'alp', True)
eg.add('ac_kwargs:hidden_sizes', params.hidden_sizes, 'hid')
activation = None
if params.activation == "relu":
activation = torch.nn.ReLU
elif params.activation == "tanh":
activation = torch.nn.Tanh
if args.cloud:
