def after_init(self, algo):
self.algo = algo
self.game_scores = torch_ext.AverageMeter(
1, self.algo.games_to_track).to(self.algo.ppo_device)
self.writer = self.algo.writer
def __init__(self, base_name, config):
self.config = config
self.env_config = config.get('env_config', {})
self.num_actors = config['num_actors']
self.env_name = config['env_name']
self.env_info = config.get('env_info')
if self.env_info is None:
self.vec_env = vecenv.create_vec_env(self.env_name, self.num_actors, **self.env_config)
self.env_info = self.vec_env.get_env_info()
self.ppo_device = config.get('device', 'cuda:0')
print('Env info:')
print(self.env_info)
self.value_size = self.env_info.get('value_size',1)
self.observation_space = self.env_info['observation_space']
self.weight_decay = config.get('weight_decay', 0.0)
self.use_action_masks = config.get('use_action_masks', False)
self.is_train = config.get('is_train', True)
self.central_value_config = self.config.get('central_value_config', None)
self.has_central_value = self.central_value_config is not None
if self.has_central_value:
self.state_space = self.env_info.get('state_space', None)
self.state_shape = None
if self.state_space.shape != None:
self.state_shape = self.state_space.shape
self.self_play_config = self.config.get('self_play_config', None)
self.has_self_play_config = self.self_play_config is not None
self.self_play = config.get('self_play', False)
self.save_freq = config.get('save_frequency', 0)
self.save_best_after = config.get('save_best_after', 100)
self.print_stats = config.get('print_stats', True)
self.rnn_states = None
self.name = base_name
self.ppo = config['ppo']
self.max_epochs = self.config.get('max_epochs', 1e6)
self.is_adaptive_lr = config['lr_schedule'] == 'adaptive'
self.linear_lr = config['lr_schedule'] == 'linear'
self.schedule_type = config.get('schedule_type', 'legacy')
if self.is_adaptive_lr:
self.lr_threshold = config['lr_threshold']
self.scheduler = schedulers.AdaptiveScheduler(self.lr_threshold)
elif self.linear_lr:
self.scheduler = schedulers.LinearScheduler(float(config['learning_rate']),
max_steps=self.max_epochs,
apply_to_entropy=config.get('schedule_entropy', False),
start_entropy_coef=config.get('entropy_coef'))
else:
self.scheduler = schedulers.IdentityScheduler()
self.e_clip = config['e_clip']
self.clip_value = config['clip_value']
self.network = config['network']
self.rewards_shaper = config['reward_shaper']
self.num_agents = self.env_info.get('agents', 1)
self.steps_num = config['steps_num']
self.seq_len = self.config.get('seq_length', 4)
self.normalize_advantage = config['normalize_advantage']
self.normalize_input = self.config['normalize_input']
self.normalize_value = self.config.get('normalize_value', False)
self.obs_shape = self.observation_space.shape
self.critic_coef = config['critic_coef']
self.grad_norm = config['grad_norm']
self.gamma = self.config['gamma']
self.tau = self.config['tau']
self.games_to_track = self.config.get('games_to_track', 100)
self.game_rewards = torch_ext.AverageMeter(self.value_size, self.games_to_track).to(self.ppo_device)
self.game_lengths = torch_ext.AverageMeter(1, self.games_to_track).to(self.ppo_device)
self.obs = None
self.games_num = self.config['minibatch_size'] // self.seq_len # it is used only for current rnn implementation
self.batch_size = self.steps_num * self.num_actors * self.num_agents
self.batch_size_envs = self.steps_num * self.num_actors
self.minibatch_size = self.config['minibatch_size']
self.mini_epochs_num = self.config['mini_epochs']
self.num_minibatches = self.batch_size // self.minibatch_size
assert(self.batch_size % self.minibatch_size == 0)
self.last_lr = self.config['learning_rate']
self.frame = 0
self.update_time = 0
self.last_mean_rewards = -100500
self.play_time = 0
self.epoch_num = 0
self.entropy_coef = self.config['entropy_coef']
self.writer = SummaryWriter('runs/' + config['name'] + datetime.now().strftime("_%d-%H-%M-%S"))
if self.normalize_value:
self.value_mean_std = RunningMeanStd((1,)).to(self.ppo_device)
self.is_tensor_obses = False
self.last_rnn_indices = None
self.last_state_indices = None
#self_play
if self.has_self_play_config:
print('Initializing SelfPlay Manager')
self.self_play_manager = SelfPlayManager(self.self_play_config, self.writer)
# features
self.algo_observer = config['features']['observer']
def base_init(self, base_name, config):
self.env_config = config.get('env_config', {})
self.num_actors = config.get('num_actors', 1)
self.env_name = config['env_name']
print("Env name:", self.env_name)
self.env_info = config.get('env_info')
if self.env_info is None:
self.vec_env = vecenv.create_vec_env(self.env_name,
self.num_actors,
**self.env_config)
self.env_info = self.vec_env.get_env_info()
self.sac_device = config.get('device', 'cuda:0')
#temporary:
self.ppo_device = self.sac_device
print('Env info:')
print(self.env_info)
self.rewards_shaper = config['reward_shaper']
self.observation_space = self.env_info['observation_space']
self.weight_decay = config.get('weight_decay', 0.0)
#self.use_action_masks = config.get('use_action_masks', False)
self.is_train = config.get('is_train', True)
self.c_loss = nn.MSELoss()
# self.c2_loss = nn.SmoothL1Loss()
self.save_best_after = config.get('save_best_after', 500)
self.print_stats = config.get('print_stats', True)
self.rnn_states = None
self.name = base_name
self.max_epochs = self.config.get('max_epochs', 1e6)
self.network = config['network']
self.rewards_shaper = config['reward_shaper']
self.num_agents = self.env_info.get('agents', 1)
self.obs_shape = self.observation_space.shape
self.games_to_track = self.config.get('games_to_track', 100)
self.game_rewards = torch_ext.AverageMeter(1, self.games_to_track).to(
self.sac_device)
self.game_lengths = torch_ext.AverageMeter(1, self.games_to_track).to(
self.sac_device)
self.obs = None
self.min_alpha = torch.tensor(np.log(1)).float().to(self.sac_device)
self.frame = 0
self.update_time = 0
self.last_mean_rewards = -100500
self.play_time = 0
self.epoch_num = 0
self.writer = SummaryWriter('runs/' + config['name'] +
datetime.now().strftime("_%d-%H-%M-%S"))
print("Run Directory:",
config['name'] + datetime.now().strftime("_%d-%H-%M-%S"))
self.is_tensor_obses = None
self.is_rnn = False
self.last_rnn_indices = None
self.last_state_indices = None
def base_init(self, base_name, config):
self.config = config
self.env_config = config.get('env_config', {})
self.num_actors = config.get('num_actors', 1)
self.env_name = config['env_name']
print("Env name:", self.env_name)
self.env_info = config.get('env_info')
if self.env_info is None:
self.vec_env = vecenv.create_vec_env(self.env_name,
self.num_actors,
**self.env_config)
self.env_info = self.vec_env.get_env_info()
self.sac_device = config.get('device', 'cuda:0')
print('Env info:')
print(self.env_info)
self.rewards_shaper = config['reward_shaper']
self.observation_space = self.env_info['observation_space']
self.weight_decay = config.get('weight_decay', 0.0)
self.use_action_masks = config.get('use_action_masks', False)
self.is_train = config.get('is_train', True)
self.central_value_config = self.config.get('central_value_config',
None)
self.has_central_value = self.central_value_config is not None
if self.has_central_value:
self.state_space = self.env_info.get('state_space', None)
self.state_shape = None
if self.state_space.shape != None:
self.state_shape = self.state_space.shape
self.self_play_config = self.config.get('self_play_config', None)
self.has_self_play_config = self.self_play_config is not None
self.self_play = config.get('self_play', False)
self.save_freq = config.get('save_frequency', 0)
self.save_best_after = config.get('save_best_after', 500)
self.print_stats = config.get('print_stats', True)
self.rnn_states = None
self.name = base_name
self.max_epochs = self.config.get('max_epochs', 1e6)
self.network = config['network']
self.rewards_shaper = config['reward_shaper']
self.num_agents = self.env_info.get('agents', 1)
self.obs_shape = self.observation_space.shape
self.games_to_track = self.config.get('games_to_track', 100)
self.game_rewards = torch_ext.AverageMeter(1, self.games_to_track).to(
self.sac_device)
self.game_lengths = torch_ext.AverageMeter(1, self.games_to_track).to(
self.sac_device)
self.obs = None
self.frame = 0
self.update_time = 0
self.last_mean_rewards = -100500
self.play_time = 0
self.epoch_num = 0
# self.writer = SummaryWriter('ant_runs/' + config['name'] + datetime.now().strftime("_%d-%H-%M-%S"))
self.writer = SummaryWriter('tested_new/' + 'shadowhand')
# self.writer = SummaryWriter('walker/'+'fixed_buffer')
print("Run Directory:",
config['name'] + datetime.now().strftime("_%d-%H-%M-%S"))
self.is_tensor_obses = None
self.curiosity_config = self.config.get('rnd_config', None)
self.has_curiosity = self.curiosity_config is not None
if self.has_curiosity:
self.curiosity_gamma = self.curiosity_config['gamma']
self.curiosity_lr = self.curiosity_config['lr']
self.curiosity_rewards = deque([], maxlen=self.games_to_track)
self.curiosity_mins = deque([], maxlen=self.games_to_track)
self.curiosity_maxs = deque([], maxlen=self.games_to_track)
self.rnd_adv_coef = self.curiosity_config.get('adv_coef', 1.0)
self.is_rnn = False
self.last_rnn_indices = None
self.last_state_indices = None