def main(args=None):
if args is None:
args = readParser()
# Initial environment
env = gym.make(args.env_name)
job_name = 'MBPO_{}_{}_{}'.format(args.env_name, args.model_type,
args.seed)
writer = SummaryWriter("tensorboard/{}".format(job_name))
writer.add_text(
'hyperparameters', "|param|value|\n|-|-|\n%s" %
('\n'.join([f"|{key}|{value}|" for key, value in vars(args).items()])))
# Set random seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
env.seed(args.seed)
# Intial agent
agent = SAC(env.observation_space.shape[0], env.action_space, args)
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = EnsembleDynamicsModel(args.num_networks,
args.num_elites,
state_size,
action_size,
args.reward_size,
args.pred_hidden_size,
use_decay=args.use_decay)
else:
env_model = construct_model(obs_dim=state_size,
act_dim=action_size,
hidden_dim=args.pred_hidden_size,
num_networks=args.num_networks,
num_elites=args.num_elites)
# Predict environments
predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
env_pool = ReplayMemory(args.replay_size)
# Initial pool for model
rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
model_steps_per_epoch = int(1 * rollouts_per_epoch)
new_pool_size = args.model_retain_epochs * model_steps_per_epoch
model_pool = ReplayMemory(new_pool_size)
# Sampler of environment
env_sampler = EnvSampler(env)
train(args, env_sampler, predict_env, agent, env_pool, model_pool, writer)
def main():
logging.basicConfig(filename=time.strftime("%Y%m%d-%H%M%S") + '_train.log',
level=logging.INFO)
args = readParser()
# Initial environment
env = gym.make(args.env_name)
# Set random seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
env.seed(args.seed)
# Intial agents ensemble
agents = []
for _ in range(args.num_agents):
agent = SAC(env.observation_space.shape[0], env.action_space, args)
agents.append(agent)
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = Ensemble_Model(args.num_networks, args.num_elites,
state_size, action_size, args.reward_size,
args.pred_hidden_size)
else:
env_model = construct_model(obs_dim=state_size,
act_dim=action_size,
hidden_dim=args.pred_hidden_size,
num_networks=args.num_networks,
num_elites=args.num_elites)
# Predict environments
predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
env_pool = ModelReplayMemory(args.replay_size)
# Initial pool for model
rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
model_steps_per_epoch = int(1 * rollouts_per_epoch)
new_pool_size = args.model_retain_epochs * model_steps_per_epoch
model_pool = ModelReplayMemory(new_pool_size)
# Sampler of environment
env_sampler = EnvSampler(env)
train(args, env_sampler, predict_env, agents, env_pool, model_pool)
def main(args=None):
if args is None:
args = readParser()
# Initial environment
env = gym.make(args.env_name)
# Set random seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
env.seed(args.seed)
# Intial agent
agent = SAC(env.observation_space.shape[0], env.action_space, args)
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = EnsembleDynamicsModel(args.num_networks,
args.num_elites,
state_size,
action_size,
args.reward_size,
args.pred_hidden_size,
use_decay=args.use_decay)
else:
env_model = construct_model(obs_dim=state_size,
act_dim=action_size,
hidden_dim=args.pred_hidden_size,
num_networks=args.num_networks,
num_elites=args.num_elites)
# Predict environments
predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
env_pool = ReplayMemory(args.replay_size)
# Initial pool for model
rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
model_steps_per_epoch = int(1 * rollouts_per_epoch)
new_pool_size = args.model_retain_epochs * model_steps_per_epoch
model_pool = ReplayMemory(new_pool_size)
# Sampler of environment
env_sampler = EnvSampler(env)
train(args, env_sampler, predict_env, agent, env_pool, model_pool)
def main(args=None):
if args is None:
args = readParser()
save_model_dir = os.path.join(args.save_dir, args.env_name,
'dynamics_model')
save_policy_dir = os.path.join(args.save_dir, args.env_name,
'policy_network')
save_env_buffer_dir = os.path.join(args.save_dir, args.env_name,
'env_buffer')
save_dynamics_buffer_dir = os.path.join(args.save_dir, args.env_name,
'dynamics_buffer')
if not os.path.exists(save_model_dir):
os.makedirs(save_model_dir)
if not os.path.exists(save_policy_dir):
os.makedirs(save_policy_dir)
if not os.path.exists(save_env_buffer_dir):
os.makedirs(save_env_buffer_dir)
if not os.path.exists(save_dynamics_buffer_dir):
os.makedirs(save_dynamics_buffer_dir)
# Initial environment
if 'Ant' in args.env_name:
args.env_name = new_env.register_mbpo_environments()[0]
print('Loaded TruncatedObs-version of the Ant environment: {}'.format(
args.env_name))
# else:
# env_name = args.env_name
env = gym.make(args.env_name)
job_name = 'MBPO_test_policy_dependent_models_{}_{}_{}'.format(
args.env_name, args.model_type, args.seed)
writer = SummaryWriter(
str(os.path.join(args.save_dir, 'tensorboard', job_name)))
writer.add_text(
'hyperparameters', "|param|value|\n|-|-|\n%s" %
('\n'.join([f"|{key}|{value}|" for key, value in vars(args).items()])))
# Set random seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
env.seed(args.seed)
# Intial agent
agent = SAC(env.observation_space.shape[0], env.action_space, args)
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = EnsembleDynamicsModel(args.num_networks,
args.num_elites,
state_size,
action_size,
args.reward_size,
args.pred_hidden_size,
use_decay=args.use_decay)
else:
env_model = construct_model(obs_dim=state_size,
act_dim=action_size,
hidden_dim=args.pred_hidden_size,
num_networks=args.num_networks,
num_elites=args.num_elites)
# Predict environments
predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
env_pool = ReplayMemory(args.replay_size)
# Initial pool for model
rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
model_steps_per_epoch = int(1 * rollouts_per_epoch)
new_pool_size = args.model_retain_epochs * model_steps_per_epoch
model_pool = ReplayMemory(new_pool_size)
# Sampler of environment
env_sampler = EnvSampler(env)
train(args, env_sampler, predict_env, agent, env_pool, model_pool, writer,
save_model_dir, save_policy_dir, save_env_buffer_dir,
save_dynamics_buffer_dir)
print('Training complete!')
print(
'---------------------------------------------------------------------'
)
print(
'Start evaluating different policies at different model checkpoints...'
)
print(
'---------------------------------------------------------------------'
)
test_policy_dependent_models(args, env, state_size, action_size,
args.save_model_freq,
args.save_model_freq * 6, save_model_dir,
save_policy_dir)
def main(args=None):
if args is None:
args = readParser()
# if not os.path.exists(args.save_model_path):
# os.makedirs(args.save_model_path)
# if not os.path.exists(args.save_policy_path):
# os.makedirs(args.save_policy_path)
# Initial environment
env = gym.make(args.env_name)
# job_name = 'MBPO_test_policy_dependent_models_{}_{}_{}'.format(args.env_name, args.model_type, args.seed)
# writer = SummaryWriter("test_policy_dependent_results_2/tensorboard/{}".format(job_name))
# writer.add_text('hyperparameters', "|param|value|\n|-|-|\n%s" % (
# '\n'.join([f"|{key}|{value}|" for key, value in vars(args).items()])))
# Set random seed
torch.manual_seed(args.seed)
np.random.seed(args.seed)
env.seed(args.seed)
# Intial agent
agent = SAC(env.observation_space.shape[0], env.action_space, args)
policy_network_checkpoint = torch.load(
'./test_policy_dependent_results_2/policy/PolicyNetwork_20.pt')
agent.policy.load_state_dict(
policy_network_checkpoint['policy_model_state_dict'])
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = EnsembleDynamicsModel(args.num_networks,
args.num_elites,
state_size,
action_size,
args.reward_size,
args.pred_hidden_size,
use_decay=args.use_decay)
else:
env_model = construct_model(obs_dim=state_size,
act_dim=action_size,
hidden_dim=args.pred_hidden_size,
num_networks=args.num_networks,
num_elites=args.num_elites)
dynamics_model_checkpoint = torch.load(
'./test_policy_dependent_results_2/dynamics_model/EnsembleDynamicsModel_20.pt'
)
env_model.ensemble_model.load_state_dict(
dynamics_model_checkpoint['dynamics_model_state_dict'])
# Predict environments
predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
env_pool = ReplayMemory(args.replay_size)
env_pool.load(
'./test_policy_dependent_results_2/env_buffer/env_buffer_20.pkl')
env_pool.position = len(env_pool.buffer)
# env_pool.buffer = np.array(env_pool.buffer)[~np.where(np.array(env_pool.buffer)==None)[0]]
# Initial pool for model
rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
model_steps_per_epoch = int(1 * rollouts_per_epoch)
new_pool_size = args.model_retain_epochs * model_steps_per_epoch
model_pool = ReplayMemory(new_pool_size)
model_pool.load(
'./test_policy_dependent_results_2/model_buffer/model_buffer_20.pkl')
model_pool.position = len(model_pool.buffer)
# model_pool.buffer = np.array(model_pool.buffer)[~np.where(np.array(model_pool.buffer)==None)[0]]
# Sampler of environment
env_sampler = EnvSampler(env)
train(args, env_sampler, predict_env, agent, env_pool, model_pool)
def main(args=None):
if args is None:
args = readParser()
if not os.path.exists(args.save_model_path):
os.makedirs(args.save_model_path)
if not os.path.exists(args.save_policy_path):
os.makedirs(args.save_policy_path)
# Initial environment
env = gym.make(args.env_name)
# job_name = 'MBPO_test_policy_dependent_models_{}_{}_{}'.format(args.env_name, args.model_type, args.seed)
# writer = SummaryWriter("test_policy_dependent_results/tensorboard/{}".format(job_name))
# writer.add_text('hyperparameters', "|param|value|\n|-|-|\n%s" % (
# '\n'.join([f"|{key}|{value}|" for key, value in vars(args).items()])))
# # Set random seed
# torch.manual_seed(args.seed)
# np.random.seed(args.seed)
# env.seed(args.seed)
# Intial agent
agent = SAC(env.observation_space.shape[0], env.action_space, args)
# Initial ensemble model
state_size = np.prod(env.observation_space.shape)
action_size = np.prod(env.action_space.shape)
if args.model_type == 'pytorch':
env_model = EnsembleDynamicsModel(args.num_networks,
args.num_elites,
state_size,
action_size,
args.reward_size,
args.pred_hidden_size,
use_decay=args.use_decay)
# else:
# env_model = construct_model(obs_dim=state_size, act_dim=action_size, hidden_dim=args.pred_hidden_size, num_networks=args.num_networks,
# num_elites=args.num_elites)
# Predict environments
# predict_env = PredictEnv(env_model, args.env_name, args.model_type)
# Initial pool for env
# env_pool = ReplayMemory(args.replay_size)
# # Initial pool for model
# rollouts_per_epoch = args.rollout_batch_size * args.epoch_length / args.model_train_freq
# model_steps_per_epoch = int(1 * rollouts_per_epoch)
# new_pool_size = args.model_retain_epochs * model_steps_per_epoch
# model_pool = ReplayMemory(new_pool_size)
# Sampler of environment
env_sampler = EnvSampler(env)
# train(args, env_sampler, predict_env, agent, env_pool, model_pool, writer)
print('Training complete!')
print(
'---------------------------------------------------------------------'
)
print(
'Start evaluating different policies at different model checkpoints...'
)
print(
'---------------------------------------------------------------------'
)
test_policy_dependent_models(args, env, state_size, action_size,
env_sampler)