def __init__(self, env, spec, aeb=(0, 0, 0)):
# essential reference variables
self.agent = None # set later
self.env = env
self.spec = spec
# agent, env, body index for multi-agent-env
self.a, self.e, self.b = self.aeb = aeb
# variables set during init_algorithm_params
self.explore_var = np.nan # action exploration: epsilon or tau
self.entropy_coef = np.nan # entropy for exploration
# debugging/logging variables, set in train or loss function
self.loss = np.nan
self.mean_entropy = np.nan
self.mean_grad_norm = np.nan
# total_reward_ma from eval for model checkpoint saves
self.best_total_reward_ma = -np.inf
self.total_reward_ma = np.nan
# dataframes to track data for analysis.analyze_session
# track training data per episode
self.train_df = pd.DataFrame(columns=[
'epi', 't', 'wall_t', 'opt_step', 'frame', 'fps', 'total_reward',
'total_reward_ma', 'loss', 'lr', 'explore_var', 'entropy_coef',
'entropy', 'grad_norm'
])
# in train@ mode, override from saved train_df if exists
if util.in_train_lab_mode() and self.spec['meta']['resume']:
train_df_filepath = util.get_session_df_path(self.spec, 'train')
if os.path.exists(train_df_filepath):
self.train_df = util.read(train_df_filepath)
self.env.clock.load(self.train_df)
# track eval data within run_eval. the same as train_df except for reward
if self.spec['meta']['rigorous_eval']:
self.eval_df = self.train_df.copy()
else:
self.eval_df = self.train_df
# the specific agent-env interface variables for a body
self.observation_space = self.env.observation_space
self.action_space = self.env.action_space
self.observable_dim = self.env.observable_dim
self.state_dim = self.observable_dim['state']
self.action_dim = self.env.action_dim
self.is_discrete = self.env.is_discrete
# set the ActionPD class for sampling action
self.action_type = policy_util.get_action_type(self.action_space)
self.action_pdtype = ps.get(spec,
f'agent.{self.a}.algorithm.action_pdtype')
if self.action_pdtype in (None, 'default'):
self.action_pdtype = policy_util.ACTION_PDS[self.action_type][0]
self.ActionPD = policy_util.get_action_pd_cls(self.action_pdtype,
self.action_type)
def __init__(self, env, agent_spec, aeb=(0, 0, 0)):
# essential reference variables
self.agent = None # set later
self.env = env
# agent, env, body index for multi-agent-env
self.a, self.e, self.b = self.aeb = aeb
# variables set during init_algorithm_params
self.explore_var = np.nan # action exploration: epsilon or tau
self.entropy_coef = np.nan # entropy for exploration
# debugging/logging variables, set in train or loss function
self.loss = np.nan
self.mean_entropy = np.nan
self.mean_grad_norm = np.nan
self.ckpt_total_reward = np.nan
self.total_reward = 0 # init to 0, but dont ckpt before end of an epi
self.total_reward_ma = np.nan
self.ma_window = 100
# store current and best reward_ma for model checkpointing and early termination if all the environments are solved
self.best_reward_ma = -np.inf
self.eval_reward_ma = np.nan
# dataframes to track data for analysis.analyze_session
# track training data per episode
self.train_df = pd.DataFrame(columns=[
'epi', 't', 'wall_t', 'opt_step', 'frame', 'fps', 'total_reward',
'total_reward_ma', 'loss', 'lr', 'explore_var', 'entropy_coef',
'entropy', 'grad_norm'
])
# track eval data within run_eval. the same as train_df except for reward
self.eval_df = self.train_df.copy()
# the specific agent-env interface variables for a body
self.observation_space = self.env.observation_space
self.action_space = self.env.action_space
self.observable_dim = self.env.observable_dim
self.state_dim = self.observable_dim['state']
self.action_dim = self.env.action_dim
self.is_discrete = self.env.is_discrete
# set the ActionPD class for sampling action
self.action_type = policy_util.get_action_type(self.action_space)
self.action_pdtype = agent_spec[self.a]['algorithm'].get(
'action_pdtype')
if self.action_pdtype in (None, 'default'):
self.action_pdtype = policy_util.ACTION_PDS[self.action_type][0]
self.ActionPD = policy_util.get_action_pd_cls(self.action_pdtype,
self.action_type)