def pivoting_rl(args): device = torch.device('cuda:'+str(args.gpu) if torch.cuda.is_available() else 'cpu') seed = 1002 # Set random seed in python std lib, numpy and pytorch set_seed(seed) vec_env = DummyVecEnvWrapper( MujocoEnv('HalfCheetah-v2') ).instantiate(parallel_envs=1, seed=seed) if args.algo == 'ddpg': model, reinforcer = get_ddpg(vec_env, device) elif args.algo == 'ppo': model, reinforcer = get_ppo(vec_env, device) else: print('Unknown algo', args.algo); assert(False) # Optimizer helper - A weird regularization settings I've copied from OpenAI code adam_optimizer = AdamFactory( lr=[1.0e-4, 1.0e-3, 1.0e-3], weight_decay=[0.0, 0.0, 0.001], eps=1.0e-4, layer_groups=True ).instantiate(model) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric('episode_rewards'), # Calculate average reward from episode ], callbacks=[StdoutStreaming()] # Print live metrics every epoch to standard output ) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 20 batches per epoch to average metrics nicely num_epochs = int(1.0e6 / 2 / 1000) # Normal handrolled training loop for i in range(1, num_epochs+1): epoch_info = EpochInfo( training_info=training_info, global_epoch_idx=i, batches_per_epoch=1000, optimizer=adam_optimizer ) reinforcer.train_epoch(epoch_info) training_info.on_train_end()
discount_factor=0.99), ) # Optimizer helper - A weird regularization settings I've copied from OpenAI code adam_optimizer = AdamFactory(lr=[1.0e-4, 1.0e-3, 1.0e-3], weight_decay=[0.0, 0.0, 0.001], eps=1.0e-4, layer_groups=True).instantiate(model) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric( 'episode_rewards'), # Calculate average reward from episode ], callbacks=[StdoutStreaming() ] # Print live metrics every epoch to standard output ) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 20 batches per epoch to average metrics nicely num_epochs = int(1.0e6 / 2 / 1000) # Normal handrolled training loop for i in range(1, num_epochs + 1): epoch_info = EpochInfo(training_info=training_info, global_epoch_idx=i,
def train_model(): """a sample training script, that creates a PPO instance and train it with bc-gym environment :return: None """ device = torch.device('cpu') seed = 1001 # Set random seed in python std lib, numpy and pytorch set_seed(seed) env_function = lambda: ColoredEgoCostmapRandomAisleTurnEnv() vec_env = DummyVecEnv([env_function]) # Again, use a helper to create a model # But because model is owned by the reinforcer, model should not be accessed using this variable # but from reinforcer.model property model = PolicyGradientModelFactory(backbone=NatureCnnTwoTowerFactory( input_width=133, input_height=133, input_channels=1)).instantiate( action_space=vec_env.action_space) # Set schedule for gradient clipping. cliprange = LinearSchedule(initial_value=0.01, final_value=0.0) # Reinforcer - an object managing the learning process reinforcer = OnPolicyIterationReinforcer( device=device, settings=OnPolicyIterationReinforcerSettings(discount_factor=0.99, batch_size=256, experience_replay=4), model=model, algo=PpoPolicyGradient(entropy_coefficient=0.01, value_coefficient=0.5, max_grad_norm=0.01, cliprange=cliprange), env_roller=StepEnvRoller( environment=vec_env, device=device, gae_lambda=0.95, number_of_steps=128, discount_factor=0.99, )) # Model optimizer optimizer = optim.Adam(reinforcer.model.parameters(), lr=1e-6, eps=1.0e-5) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric( 'episode_rewards'), # Calculate average reward from episode ], callbacks=[ StdoutStreaming( ), # Print live metrics every epoch to standard output FrameTracker( 1.1e8 ) # We need frame tracker to track the progress of learning ]) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 10 batches per epoch to average metrics nicely # Rollout size is 8 environments times 128 steps num_epochs = int(1.1e8 / (128 * 1) / 10) # Normal handrolled training loop eval_results = [] for i in range(1, num_epochs + 1): epoch_info = EpochInfo(training_info=training_info, global_epoch_idx=i, batches_per_epoch=10, optimizer=optimizer) reinforcer.train_epoch(epoch_info) eval_result = evaluate_model(model, vec_env, device, takes=1) eval_results.append(eval_result) if i % 100 == 0: torch.save(model.state_dict(), 'tmp_checkout.data') with open('tmp_eval_results.pkl', 'wb') as f: pickle.dump(eval_results, f, 0) training_info.on_train_end()
def breakout_a2c(): device = torch.device('cuda:0') seed = 1001 # Set random seed in python std lib, numpy and pytorch set_seed(seed) # Create 16 environments evaluated in parallel in sub processess with all usual DeepMind wrappers # These are just helper functions for that vec_env = SubprocVecEnvWrapper(ClassicAtariEnv('BreakoutNoFrameskip-v4'), frame_history=4).instantiate( parallel_envs=16, seed=seed) # Again, use a helper to create a model # But because model is owned by the reinforcer, model should not be accessed using this variable # but from reinforcer.model property model = PolicyGradientModelFactory(backbone=NatureCnnFactory( input_width=84, input_height=84, input_channels=4)).instantiate( action_space=vec_env.action_space) # Reinforcer - an object managing the learning process reinforcer = OnPolicyIterationReinforcer( device=device, settings=OnPolicyIterationReinforcerSettings( discount_factor=0.99, batch_size=256, ), model=model, algo=A2CPolicyGradient(entropy_coefficient=0.01, value_coefficient=0.5, max_grad_norm=0.5), env_roller=StepEnvRoller( environment=vec_env, device=device, number_of_steps=5, discount_factor=0.99, )) # Model optimizer optimizer = optim.RMSprop(reinforcer.model.parameters(), lr=7.0e-4, eps=1e-3) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric( 'episode_rewards'), # Calculate average reward from episode ], callbacks=[StdoutStreaming() ] # Print live metrics every epoch to standard output ) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 100 batches per epoch to average metrics nicely num_epochs = int(1.1e7 / (5 * 16) / 100) # Normal handrolled training loop for i in range(1, num_epochs + 1): epoch_info = EpochInfo(training_info=training_info, global_epoch_idx=i, batches_per_epoch=100, optimizer=optimizer) reinforcer.train_epoch(epoch_info) training_info.on_train_end()
) # Optimizer helper - A weird regularization settings I've copied from OpenAI code adam_optimizer = AdamFactory( lr=[1.0e-4, 1.0e-3, 1.0e-3], weight_decay=[0.0, 0.0, 0.001], eps=1.0e-4, layer_groups=True ).instantiate(model) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric('episode_rewards'), # Calculate average reward from episode ], callbacks=[StdoutStreaming()] # Print live metrics every epoch to standard output ) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 20 batches per epoch to average metrics nicely num_epochs = int(1.0e6 / 64 / 20) # Normal handrolled training loop for i in range(1, num_epochs+1): epoch_info = EpochInfo( training_info=training_info, global_epoch_idx=i,
def qbert_ppo(): device = torch.device('cuda:0') seed = 1001 # Set random seed in python std lib, numpy and pytorch set_seed(seed) # Create 16 environments evaluated in parallel in sub processess with all usual DeepMind wrappers # These are just helper functions for that vec_env = SubprocVecEnvWrapper(ClassicAtariEnv('QbertNoFrameskip-v4'), frame_history=4).instantiate( parallel_envs=8, seed=seed) # Again, use a helper to create a model # But because model is owned by the reinforcer, model should not be accessed using this variable # but from reinforcer.model property model = StochasticPolicyModelFactory( input_block=ImageToTensorFactory(), backbone=NatureCnnFactory( input_width=84, input_height=84, input_channels=4)).instantiate(action_space=vec_env.action_space) # Set schedule for gradient clipping. cliprange = LinearSchedule(initial_value=0.1, final_value=0.0) # Reinforcer - an object managing the learning process reinforcer = OnPolicyIterationReinforcer( device=device, settings=OnPolicyIterationReinforcerSettings(batch_size=256, experience_replay=4, number_of_steps=128), model=model, algo=PpoPolicyGradient(entropy_coefficient=0.01, value_coefficient=0.5, max_grad_norm=0.5, discount_factor=0.99, gae_lambda=0.95, cliprange=cliprange), env_roller=StepEnvRoller( environment=vec_env, device=device, )) # Model optimizer optimizer = optim.Adam(reinforcer.model.parameters(), lr=2.5e-4, eps=1.0e-5) # Overall information store for training information training_info = TrainingInfo( metrics=[ EpisodeRewardMetric( 'episode_rewards'), # Calculate average reward from episode ], callbacks=[ StdoutStreaming( ), # Print live metrics every epoch to standard output FrameTracker( 1.1e7 ) # We need frame tracker to track the progress of learning ]) # A bit of training initialization bookkeeping... training_info.initialize() reinforcer.initialize_training(training_info) training_info.on_train_begin() # Let's make 10 batches per epoch to average metrics nicely # Rollout size is 8 environments times 128 steps num_epochs = int(1.1e7 / (128 * 8) / 10) # Normal handrolled training loop for i in range(1, num_epochs + 1): epoch_info = EpochInfo(training_info=training_info, global_epoch_idx=i, batches_per_epoch=10, optimizer=optimizer) reinforcer.train_epoch(epoch_info) training_info.on_train_end()