def build_experiment():
requirements = xm.Requirements(ram=10 * xm.GiB)
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=FLAGS.priority,
requirements=requirements,
overrides=overrides,
)
save_dir = os.path.join(
FLAGS.save_dir.format(CELL=FLAGS.cell),
'{EXP}_gamma{GAMMA}'.format(EXP=FLAGS.exp_name, GAMMA=FLAGS.gamma))
executable = xm.BuildTarget(
'//third_party/py/dice_rl/scripts:run_neural_dice',
build_flags=['-c', 'opt', '--copt=-mavx'],
args=[
('env_name', FLAGS.env_name),
('gamma', FLAGS.gamma),
('save_dir', save_dir),
('load_dir', FLAGS.load_dir),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
max_traj_dict = {
'grid': 100,
'taxi': 200,
'Reacher-v2': 40,
'reacher': 200,
'cartpole': 250,
}
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
hyper.sweep('zero_reward', hyper.categorical([False])),
hyper.sweep('norm_regularizer', hyper.discrete([0.0, 1.0])),
hyper.sweep('zeta_pos', hyper.categorical([True, False])),
hyper.sweep('primal_form', hyper.categorical([False])),
hyper.sweep('num_steps', hyper.discrete([200000])),
hyper.sweep('f_exponent', hyper.discrete([2.0])),
hyper.zipit([
hyper.sweep('primal_regularizer', hyper.discrete([0.0, 1.0])),
hyper.sweep('dual_regularizer', hyper.discrete([1.0, 0.0])),
]),
hyper.zipit([
hyper.sweep('nu_learning_rate', hyper.discrete([0.0001])),
hyper.sweep('zeta_learning_rate', hyper.discrete([0.0001])),
]),
hyper.sweep('alpha', hyper.discrete([0.0])),
hyper.sweep('num_trajectory', hyper.discrete([100])),
hyper.sweep(
'max_trajectory_length',
hyper.discrete([
100 #max_traj_dict[FLAGS.env_name]
])),
])
experiment = xm.ParameterSweep(
executable, parameters, max_parallel_work_units=2000)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
save_dir = os.path.join(FLAGS.save_dir, FLAGS.exp_name)
requirements = xm.Requirements(ram=10 * xm.GiB)
if FLAGS.worker_ram_fs_gb is not None:
requirements.tmp_ram_fs_size = FLAGS.worker_ram_fs_gb * xm.GiB
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
if FLAGS.avx2:
overrides.requirements.constraints = AVX2_CONSTRAINTS
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=115,
requirements=requirements,
overrides=overrides,
)
num_trajectory = 200
executable = xm.BuildTarget(
'//third_party/py/dice_rl/scripts:run_tabular_coin_dice',
build_flags=AVX2_BUILD_FLAGS if FLAGS.avx2 else AVX_BUILD_FLAGS,
args=[
('env_name', FLAGS.env_name),
('load_dir', FLAGS.load_dir),
('save_dir', save_dir),
('num_steps', 100),
('num_trajectory', num_trajectory),
('max_trajectory_length', 1),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
num_samples = num_trajectory
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
hyper.sweep('alpha', [0.0]),
hyper.sweep('gamma', [0.0]),
hyper.sweep('divergence_limit', [
1.0 * CHI2_PERCENTILE[p] / num_samples
for p in [0, 50, 60, 70, 80, 90, 95]
]),
hyper.sweep('algae_alpha', [0.01]),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
save_dir = os.path.join(FLAGS.save_dir, FLAGS.exp_name)
requirements = xm.Requirements(ram=10 * xm.GiB)
if FLAGS.worker_ram_fs_gb is not None:
requirements.tmp_ram_fs_size = FLAGS.worker_ram_fs_gb * xm.GiB
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
if FLAGS.avx2:
overrides.requirements.constraints = AVX2_CONSTRAINTS
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=115,
requirements=requirements,
overrides=overrides,
)
executable = xm.BuildTarget(
'//third_party/py/dice_rl/google/scripts:run_q_estimator',
build_flags=AVX2_BUILD_FLAGS if FLAGS.avx2 else AVX_BUILD_FLAGS,
args=[
('env_name', FLAGS.env_name),
('load_dir', FLAGS.load_dir),
('save_dir', save_dir),
('num_trajectory', 10000),
('max_trajectory_length', 1),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
#hyper.sweep('alpha', [0.0, 0.33, 0.66, 1.]),
hyper.sweep('alpha', [0.0, 0.1, 0.2, 0.9]),
hyper.sweep('gamma', [0.0]),
hyper.sweep('limit_episodes', [2, 5, 10, 20, 50, 100, 200, 500,
1000, 2000, 5000, 10000]),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
requirements = xm.Requirements()
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=FLAGS.priority,
requirements=requirements,
)
gamma = 0.99 if FLAGS.env_name != 'bandit' else 0.0
save_dir = os.path.join(
FLAGS.save_dir.format(CELL=FLAGS.cell),
'{EXP}_gamma{GAMMA}'.format(EXP=FLAGS.exp_name, GAMMA=gamma))
executable = xm.BuildTarget(
'//third_party/py/dice_rl/scripts:run_tabular_bayes_dice',
build_flags=['-c', 'opt', '--copt=-mavx'],
args=[('env_name', FLAGS.env_name), ('gamma', gamma),
('save_dir', save_dir), ('load_dir', FLAGS.load_dir),
('num_steps', 50000),
('solve_for_state_action_ratio',
False if FLAGS.env_name == 'taxi' else True)],
platform=xm.Platform.CPU,
runtime=runtime_worker)
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
hyper.sweep('alpha', hyper.discrete([0.])),
hyper.sweep('alpha_target',
hyper.discrete([i / 100 for i in range(86, 95)])),
hyper.sweep('kl_regularizer', hyper.discrete([5.])),
hyper.sweep('num_trajectory', hyper.discrete([5, 10, 25, 50])),
hyper.sweep(
'max_trajectory_length',
hyper.discrete([
#1, # bandit
#500, # taxi
100, # frozenlake
])),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
requirements = xm.Requirements()
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=FLAGS.priority,
requirements=requirements,
overrides=overrides,
)
save_dir = os.path.join(
FLAGS.save_dir.format(CELL=FLAGS.cell),
'{EXP}_gamma{GAMMA}'.format(EXP=FLAGS.exp_name, GAMMA=FLAGS.gamma))
executable = xm.BuildTarget(
'//third_party/py/dice_rl/scripts:run_neural_bayes_dice',
build_flags=['-c', 'opt', '--copt=-mavx'],
args=[
('env_name', FLAGS.env_name),
('gamma', FLAGS.gamma),
('save_dir', save_dir),
('load_dir', FLAGS.load_dir),
('num_steps', 50000),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
hyper.sweep('kl_regularizer', hyper.discrete([5.])),
hyper.sweep('alpha', hyper.discrete([i / 10 for i in range(6)])),
hyper.sweep('alpha_target',
hyper.discrete([0.75, 0.8, 0.85, 0.9, 0.95])),
hyper.sweep('num_trajectory', hyper.discrete([10, 25, 50, 100])),
hyper.sweep('max_trajectory_length', hyper.discrete([100])),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
"""Create the jobs/config and return the constructed experiment."""
# ====== Argument creation ======
model_dir = FLAGS.model_dir.format(
cell=FLAGS.cell,
user=getpass.getuser(),
trial=FLAGS.trial,
)
# ====== Jobs and runtime creation ======
# Job: worker
requirements = xm.Requirements(gpu_types=[xm.GpuType.V100], )
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=FLAGS.priority,
requirements=requirements,
)
exec_worker = xm.BuildTarget(
'//experimental/users/gjt/his:mnist',
name='worker',
args=dict(
gfs_user=FLAGS.gfs_user,
logdir=model_dir,
mode='train',
),
platform=xm.Platform.GPU,
runtime=runtime_worker,
)
# Job: eval
runtime_eval = xm.Borg(
cell=FLAGS.cell,
priority=FLAGS.priority,
)
exec_eval = xm.BuildTarget(
'//experimental/users/gjt/his:mnist',
name='eval',
args=dict(
gfs_user=FLAGS.gfs_user,
logdir=model_dir,
mode='eval',
split='train,valid,test',
num_iwae_samples='1,1,1000',
),
platform=xm.Platform.GPU, # Do we need GPU for eval?
runtime=runtime_eval,
)
# ====== Executable experiment creation ======
list_executables = []
list_executables.append(xm_helper.build_single_job(exec_worker))
list_executables.append(xm_helper.build_single_job(exec_eval))
experiment = xm.ParallelExecutable(list_executables, name='his_service')
# Build experiments
hyper_parameters = {}
# SNIS vs LARS
hyper_parameters['snis_vs_lars'] = hyper.product([
hyper.chainit([
hyper.product([
hyper.fixed('proposal', 'gaussian', length=1),
hyper.fixed('model', 'bernoulli_vae', length=1),
]),
hyper.product([
hyper.fixed('proposal', 'bernoulli_vae', length=1),
hyper.fixed('model', 'nis', length=1),
]),
hyper.product([
hyper.fixed('proposal', 'nis', length=1),
hyper.fixed('model', 'bernoulli_vae', length=1),
]),
]),
hyper.sweep('run', hyper.discrete([0])),
hyper.fixed('dataset', 'static_mnist', length=1),
hyper.fixed('reparameterize_proposal', False, length=1),
hyper.fixed('anneal_kl_step', 100000, length=1),
])
# Continuous comparisons: HIS, NIS, VAE
hyper_parameters['continuous'] = hyper.product([
hyper.chainit([
hyper.product([
hyper.fixed('proposal', 'gaussian', length=1),
hyper.fixed('model', 'gaussian_vae', length=1),
]),
hyper.product([
hyper.fixed('proposal', 'gaussian_vae', length=1),
hyper.fixed('model', 'nis', length=1),
]),
hyper.product([
hyper.fixed('proposal', 'gaussian', length=1),
hyper.fixed('model', 'hisvae', length=1),
hyper.sweep('his_T', hyper.discrete([5, 10, 15])),
]),
]),
hyper.sweep('run', hyper.discrete([0])),
hyper.fixed('dataset', 'jittered_mnist', length=1),
hyper.fixed('reparameterize_proposal', True, length=1),
hyper.fixed('squash', True, length=1),
])
hyper_parameters['celeba'] = hyper.product([
hyper.chainit([
hyper.product([
hyper.fixed('proposal', 'gaussian', length=1),
hyper.fixed('model', 'conv_gaussian_vae', length=1),
]),
]),
hyper.sweep('run', hyper.discrete([0])),
hyper.fixed('dataset', 'jittered_celeba', length=1),
hyper.fixed('reparameterize_proposal', True, length=1),
hyper.fixed('squash', True, length=1),
hyper.fixed('latent_dim', 16, length=1),
hyper.fixed('batch_size', 36, length=1),
])
experiment = xm.ParameterSweep(experiment,
hyper_parameters[FLAGS.exp_type])
experiment = xm.WithTensorBoard(experiment, model_dir)
return experiment
def build_experiment():
save_dir = os.path.join(FLAGS.save_dir, FLAGS.exp_name)
requirements = xm.Requirements(ram=10 * xm.GiB)
if FLAGS.worker_ram_fs_gb is not None:
requirements.tmp_ram_fs_size = FLAGS.worker_ram_fs_gb * xm.GiB
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
if FLAGS.avx2:
overrides.requirements.constraints = AVX2_CONSTRAINTS
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=115,
requirements=requirements,
overrides=overrides,
)
executable = xm.BuildTarget(
'//third_party/py/dice_rl/google/scripts:run_neural_robust',
build_flags=['-c', 'opt', '--copt=-mavx'],
args=[
('env_name', FLAGS.env_name),
('load_dir', FLAGS.load_dir),
('save_dir', save_dir),
('num_steps', 100000),
('batch_size', 128),
#('num_trajectory', 200),
#('max_trajectory_length', 250),
('num_trajectory', 100),
('max_trajectory_length', 100),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
num_samples = 100 * 100
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
#hyper.sweep('seed', [0]),
#hyper.sweep('bootstrap_seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
hyper.sweep('alpha', [0.0]), #, 0.33, 0.66]),
hyper.sweep('nu_learning_rate', [0.001, 0.0003, 0.0001]),
hyper.sweep('zeta_learning_rate', [0.001, 0.0003, 0.0001]),
#hyper.sweep('nu_learning_rate', [0.0001, 0.0003]),
#hyper.sweep('zeta_learning_rate', [0.0003]),
#hyper.sweep('nu_learning_rate', [0.001]),
#hyper.sweep('zeta_learning_rate', [0.0001, 0.0003]),
hyper.sweep('gamma', [0.99]),
hyper.zipit([
hyper.sweep('nu_regularizer', [0.0]),
hyper.sweep('zeta_regularizer', [0.0])
]),
#hyper.sweep('divergence_limit', [0.002, 0.005, 0.01]),
#hyper.sweep('algae_alpha', [0.001]),
hyper.sweep('divergence_limit', [
0.5 * CHI2_PERCENTILE[p] / num_samples
for p in [0, 50, 60, 70, 80]
]),
hyper.sweep('algae_alpha', [0.01]),
hyper.sweep('primal_form', [True]),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment
def build_experiment():
save_dir = os.path.join(FLAGS.save_dir, FLAGS.exp_name)
requirements = xm.Requirements(ram=10 * xm.GiB)
if FLAGS.worker_ram_fs_gb is not None:
requirements.tmp_ram_fs_size = FLAGS.worker_ram_fs_gb * xm.GiB
overrides = xm.BorgOverrides()
overrides.requirements.autopilot_params = ({'min_cpu': 1})
if FLAGS.avx2:
overrides.requirements.constraints = AVX2_CONSTRAINTS
runtime_worker = xm.Borg(
cell=FLAGS.cell,
priority=115,
requirements=requirements,
overrides=overrides,
)
executable = xm.BuildTarget(
'//third_party/py/dice_rl/google/scripts:run_importance_sampling_ci',
build_flags=['-c', 'opt', '--copt=-mavx'],
args=[
('gfs_user', 'brain-ofirnachum'),
('env_name', FLAGS.env_name),
('load_dir', FLAGS.load_dir),
('num_trajectory_data', FLAGS.num_trajectory_data),
('save_dir', save_dir),
('num_steps', 10000),
('alpha', -1.0),
('ci_method', FLAGS.ci_method),
('mode', FLAGS.mode),
('tabular_obs', FLAGS.tabular_obs),
('use_trained_policy', False),
('use_doubly_robust', False),
],
platform=xm.Platform.CPU,
runtime=runtime_worker)
parameters = hyper.product([
hyper.sweep('seed', hyper.discrete(list(range(FLAGS.num_seeds)))),
## Reacher
#hyper.sweep('gamma', [0.99]),
#hyper.sweep('num_trajectory', [25]),
#hyper.sweep('max_trajectory_length', [100]),
## FrozenLake
# hyper.sweep('gamma', [0.99]),
# hyper.sweep('num_trajectory', [50, 100, 200, 500, 1000]),
# hyper.sweep('max_trajectory_length', [100]),
## SmallTree
hyper.sweep('gamma', [0.0]),
hyper.sweep('num_trajectory', [50, 100, 200]),
hyper.sweep('max_trajectory_length', [1]),
## Taxi
# hyper.sweep('gamma', [0.99]),
# hyper.sweep('num_trajectory', [20, 50, 100]),
# hyper.sweep('max_trajectory_length', [500]),
## universally needed
hyper.sweep('delta', [0.5, 0.6, 0.7, 0.8, 0.9, 0.95]),
])
experiment = xm.ParameterSweep(executable, parameters)
experiment = xm.WithTensorBoard(experiment, save_dir)
return experiment