stub(globals())
env = TfEnv(normalize(CartpoleEnv()))
policy = GaussianLSTMPolicy(
name="policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=4000,
max_path_length=100,
n_itr=10,
discount=0.99,
step_size=0.01,
optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(base_eps=1e-5))
)
run_experiment_lite(
algo.train(),
n_parallel=4,
seed=1,
)
n_itr=n_itr,
optimizer_args={
'init_learning_rate': step_sizes[step_i],
'tf_optimizer_args': {
'learning_rate': 0.5 * step_sizes[step_i]
},
'tf_optimizer_cls': tf.train.GradientDescentOptimizer
})
run_experiment_lite(
algo.train(),
# Number of parallel workers for sampling
n_parallel=4,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="last",
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=4,
exp_prefix='trpopoint2d_test',
exp_name='test',
#plot=True,
)
import pdb
pdb.set_trace()
# get return from the experiment
with open('data/local/trpopoint2d-test/test/progress.csv', 'r') as f:
reader = csv.reader(f, delimiter=',')
i = 0
row = None
returns = []
for row in reader:
reset_arg=goal,
optimizer_args={
'init_learning_rate': step_sizes[step_i],
'tf_optimizer_args': {
'learning_rate': 0.5 * step_sizes[step_i]
},
'tf_optimizer_cls': tf.train.GradientDescentOptimizer
})
run_experiment_lite(
algo.train(),
# Number of parallel workers for sampling
n_parallel=4,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="all",
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=goal_i,
exp_prefix='cheetahdirec_test',
exp_name='test' + str(run_id),
plot=True,
)
# get return from the experiment
with open(
'data/local/cheetahdirec-test/test' + str(run_id) +
'/progress.csv', 'r') as f:
reader = csv.reader(f, delimiter=',')
i = 0
row = None
returns = []
for row in reader:
reset_arg=goal,
optimizer_args={
'init_learning_rate': step_sizes[step_i],
'tf_optimizer_args': {
'learning_rate': 0.5 * step_sizes[step_i]
},
'tf_optimizer_cls': tf.train.GradientDescentOptimizer
})
run_experiment_lite(
algo.train(),
# Number of parallel workers for sampling
n_parallel=4,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="all",
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=
2, #1 # don't set the seed for oracle, since it's already deterministic.
exp_prefix='cheetah_test',
exp_name='test' + str(run_id),
plot=True,
)
# get return from the experiment
import csv
with open(
'data/local/cheetah-test/test' + str(run_id) + '/progress.csv',
'r') as f:
reader = csv.reader(f, delimiter=',')
i = 0
row = None
returns = []
def run_experiment(argv):
# -------------------- Parse Arguments -----------------------------------
parser = argparse.ArgumentParser()
parser.add_argument(
'--mode',
type=str,
default='local',
help='Mode for running the experiments - local: runs on local machine, '
'ec2: runs on AWS ec2 cluster (requires a proper configuration file)')
args = parser.parse_args(argv[1:])
# -------------------- Define Variants -----------------------------------
vg = VariantGenerator()
vg.add('env',
['HalfCheetahEnvRandParams'
]) #Reacher5DofMAMLEnvRandParams HalfCheetahMAMLEnvRandParams
vg.add('log_scale_limit', [0.1, 0.3, 0.5])
vg.add('fast_lr', [0.1])
vg.add('meta_batch_size', [40])
vg.add('num_grad_updates', [1])
vg.add('meta_step_size', [0.01])
vg.add('fast_batch_size', [20])
vg.add('seed', [1, 11]) #TODO add [21, 31, 41]
vg.add('discount', [0.99])
vg.add('n_iter', [500])
vg.add('path_length', [100])
vg.add('hidden_nonlinearity', ['tanh'])
vg.add('hidden_sizes', [(100, 100)])
vg.add('trainable_step_size', [True, False])
vg.add('bias_transform', [False])
vg.add('policy', ['MAMLImprovedGaussianMLPPolicy'])
variants = vg.variants()
# ----------------------- AWS conficuration ---------------------------------
if args.mode == 'ec2':
subnets = cheapest_subnets(ec2_instance, num_subnets=3)
info = config.INSTANCE_TYPE_INFO[ec2_instance]
config.AWS_INSTANCE_TYPE = ec2_instance
config.AWS_SPOT_PRICE = str(info["price"])
print("\n" + "**********" * 10 +
"\nexp_prefix: {}\nvariants: {}".format('TRPO', len(variants)))
print(
'Running on type {}, with price {}, on the subnets: '.format(
config.AWS_INSTANCE_TYPE,
config.AWS_SPOT_PRICE,
), str(subnets))
if args.mode == 'ec2':
info = config.INSTANCE_TYPE_INFO[ec2_instance]
n_parallel = int(info["vCPU"] /
2) # make the default 4 if not using ec2
else:
n_parallel = 12
# ----------------------- TRAINING ---------------------------------------
exp_ids = random.sample(range(1, 1000), len(variants))
for v, exp_id in zip(variants, exp_ids):
exp_name = "trpo_maml_train_env_%s_%.3f_%.3f_%i_id_%i" % (
v['env'], v['log_scale_limit'], v['meta_step_size'], v['seed'],
exp_id)
v = instantiate_class_stings(v)
if args.mode == 'ec2':
# configure instance
subnet = random.choice(subnets)
config.AWS_REGION_NAME = subnet[:-1]
config.AWS_KEY_NAME = config.ALL_REGION_AWS_KEY_NAMES[
config.AWS_REGION_NAME]
config.AWS_IMAGE_ID = config.ALL_REGION_AWS_IMAGE_IDS[
config.AWS_REGION_NAME]
config.AWS_SECURITY_GROUP_IDS = \
config.ALL_REGION_AWS_SECURITY_GROUP_IDS[
config.AWS_REGION_NAME]
run_experiment_lite(
run_train_task,
exp_prefix=EXP_PREFIX,
exp_name=exp_name,
# Number of parallel workers for sampling
n_parallel=n_parallel,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="last",
periodic_sync=True,
sync_s3_pkl=True,
sync_s3_log=True,
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
pre_commands=[
"yes | pip install --upgrade pip",
"yes | pip install tensorflow=='1.6.0'",
"yes | pip install --upgrade cloudpickle"
],
seed=v["seed"],
python_command="python3",
mode=args.mode,
use_cloudpickle=True,
variant=v,
)
max_path_length=max_path_length,
meta_batch_size=v['meta_batch_size'],
num_grad_updates=num_grad_updates,
n_itr=800,
use_maml=use_maml,
step_size=v['meta_step_size'],
plot=False,
)
direc = 'direc' if direc else ''
run_experiment_lite(
algo.train(),
exp_prefix='trpo_maml_cheetah' + direc + str(max_path_length),
exp_name='maml'+str(int(use_maml))+'_fbs'+str(v['fast_batch_size'])+'_mbs'+str(v['meta_batch_size'])+'_flr_' + str(v['fast_lr']) + '_mlr' + str(v['meta_step_size']),
# Number of parallel workers for sampling
n_parallel=8,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="gap",
snapshot_gap=25,
sync_s3_pkl=True,
python_command=sys.executable,
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=v["seed"],
mode="local",
#mode="ec2",
variant=v,
# plot=True,
# terminate_machine=False,
)
def run_evaluation(argv):
# -------------------- Parse Arguments -----------------------------------
parser = argparse.ArgumentParser()
parser.add_argument(
'exp_prefix_dir',
type=str,
help='path to dump dir which contains folders with '
'the train results i.e. params.pkl and variant.json file')
parser.add_argument(
'--mode',
type=str,
default='local',
help='Mode for running the experiments - local: runs on local machine, '
'ec2: runs on AWS ec2 cluster (requires a proper configuration file)')
parser.add_argument(
'--n_parallel',
type=int,
default=1,
help=
'Number of parallel workers to perform rollouts. 0 => don\'t start any workers'
)
parser.add_argument('--num_sampled_envs',
type=int,
default=5,
help='number or environments with samples parameters')
args = parser.parse_args(argv[1:])
# ----------------------- EVALUATION ---------------------------------------
exp_prefix = os.path.basename(args.exp_prefix_dir)
eval_exp_prefix = exp_prefix + '-eval'
evaluation_runs = eval.prepare_evaluation_runs(
args.exp_prefix_dir,
EXP_PREFIX,
num_sampled_envs=args.num_sampled_envs)
# ----------------------- AWS conficuration ---------------------------------
if args.mode == 'ec2':
subnets = cheapest_subnets(ec2_instance, num_subnets=3)
info = config.INSTANCE_TYPE_INFO[ec2_instance]
config.AWS_INSTANCE_TYPE = ec2_instance
config.AWS_SPOT_PRICE = str(info["price"])
print("\n" + "**********" * 10 +
"\nexp_prefix: {}\nvariants: {}".format('TRPO',
len(evaluation_runs)))
print(
'Running on type {}, with price {}, on the subnets: '.format(
config.AWS_INSTANCE_TYPE,
config.AWS_SPOT_PRICE,
), str(subnets))
for eval_exp_name, v in evaluation_runs:
if args.mode == 'ec2':
subnet = random.choice(subnets)
config.AWS_REGION_NAME = subnet[:-1]
config.AWS_KEY_NAME = config.ALL_REGION_AWS_KEY_NAMES[
config.AWS_REGION_NAME]
config.AWS_IMAGE_ID = config.ALL_REGION_AWS_IMAGE_IDS[
config.AWS_REGION_NAME]
config.AWS_SECURITY_GROUP_IDS = \
config.ALL_REGION_AWS_SECURITY_GROUP_IDS[
config.AWS_REGION_NAME]
run_experiment_lite(
run_eval_task,
exp_prefix=eval_exp_prefix,
exp_name=eval_exp_name,
# Number of parallel workers for sampling
n_parallel=args.n_parallel,
# Only keep the snapshot parameters for the last iteration
snapshot_mode="last",
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=v["seed"],
python_command="python3", #TODO
mode=args.mode,
use_cloudpickle=True,
periodic_sync=True,
variant=v,
# plot=True,
# terminate_machine=False,
)
elif 'linear' in bas:
baseline = LinearFeatureBaseline(env_spec=env.spec)
else:
baseline = GaussianMLPBaseline(env_spec=env.spec)
algo = MAMLTRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=fast_batch_size, # number of trajs for grad update
max_path_length=max_path_length,
meta_batch_size=meta_batch_size,
num_grad_updates=num_grad_updates,
n_itr=100,
use_maml=use_maml,
step_size=meta_step_size,
plot=False,
)
run_experiment_lite(
algo.train(),
n_parallel=1,
snapshot_mode="last",
python_command=interpreter_path,
seed=1,
exp_prefix='trpo_maml_point100',
exp_name='trpomaml' + str(int(use_maml)) + '_fbs' +
str(fast_batch_size) + '_mbs' + str(meta_batch_size) +
'_flr_' + str(fast_learning_rate) + 'metalr_' +
str(meta_step_size) + '_step1' + str(num_grad_updates),
plot=False,
)
if oracle:
exp_name = 'oracleenv'
else:
exp_name = 'randenv'
if direc:
exp_prefix = 'trpo_maml_cheetahdirec' + str(max_path_length)
else:
exp_prefix = 'bugfix_trpo_maml_cheetah' + str(max_path_length)
run_experiment_lite(
algo.train(),
exp_prefix=exp_prefix,
exp_name=exp_name,
# Number of parallel workers for sampling
n_parallel=1,
# Only keep the snapshot parameters for the last iteration
#snapshot_mode="last",
snapshot_mode="gap",
snapshot_gap=25,
sync_s3_pkl=True,
python_command=sys.executable,
# Specifies the seed for the experiment. If this is not provided, a random seed
# will be used
seed=v["seed"],
mode="local",
#mode="ec2",
variant=v,
# plot=True,
# terminate_machine=False,
)