def forward_tail(self, core_output, with_action_distribution=False): core_outputs = core_output.chunk(len(self.cores), dim=1) # first core output corresponds to the actor action_distribution_params, action_distribution = self.action_parameterization( core_outputs[0]) # for non-trivial action spaces it is faster to do these together actions, log_prob_actions = sample_actions_log_probs( action_distribution) # second core output corresponds to the critic values = self.critic_linear(core_outputs[1]) result = AttrDict( dict( actions=actions, action_logits=action_distribution_params, log_prob_actions=log_prob_actions, values=values, )) if with_action_distribution: result.action_distribution = action_distribution return result
def get_obs_shape(obs_space): obs_shape = AttrDict() if hasattr(obs_space, 'spaces'): for key, space in obs_space.spaces.items(): obs_shape[key] = space.shape else: obs_shape.obs = obs_space.shape return obs_shape
def _handle_policy_steps(self, timing): with torch.no_grad(): with timing.add_time('deserialize'): observations = AttrDict() rnn_states = [] traj_tensors = self.shared_buffers.tensors_individual_transitions for request in self.requests: actor_idx, split_idx, request_data = request for env_idx, agent_idx, traj_buffer_idx, rollout_step in request_data: index = actor_idx, split_idx, env_idx, agent_idx, traj_buffer_idx, rollout_step dict_of_lists_append(observations, traj_tensors['obs'], index) rnn_states.append(traj_tensors['rnn_states'][index]) self.total_num_samples += 1 with timing.add_time('stack'): for key, x in observations.items(): observations[key] = torch.stack(x) rnn_states = torch.stack(rnn_states) num_samples = rnn_states.shape[0] with timing.add_time('obs_to_device'): for key, x in observations.items(): device, dtype = self.actor_critic.device_and_type_for_input_tensor(key) observations[key] = x.to(device).type(dtype) rnn_states = rnn_states.to(self.device).float() with timing.add_time('forward'): policy_outputs = self.actor_critic(observations, rnn_states) with timing.add_time('to_cpu'): for key, output_value in policy_outputs.items(): policy_outputs[key] = output_value.cpu() with timing.add_time('format_outputs'): policy_outputs.policy_version = torch.empty([num_samples]).fill_(self.latest_policy_version) # concat all tensors into a single tensor for performance output_tensors = [] for policy_output in self.shared_buffers.policy_outputs: tensor_name = policy_output.name output_value = policy_outputs[tensor_name].float() if len(output_value.shape) == 1: output_value.unsqueeze_(dim=1) output_tensors.append(output_value) output_tensors = torch.cat(output_tensors, dim=1) with timing.add_time('postprocess'): self._enqueue_policy_outputs(self.requests, output_tensors) self.requests = []
def print_stats(self, fps, sample_throughput, total_env_steps): fps_str = [] for interval, fps_value in zip(self.avg_stats_intervals, fps): fps_str.append( f'{int(interval * self.report_interval)} sec: {fps_value:.1f}') fps_str = f'({", ".join(fps_str)})' samples_per_policy = ', '.join( [f'{p}: {s:.1f}' for p, s in sample_throughput.items()]) lag_stats = self.policy_lag[0] lag = AttrDict() for key in ['min', 'avg', 'max']: lag[key] = lag_stats.get(f'version_diff_{key}', -1) policy_lag_str = f'min: {lag.min:.1f}, avg: {lag.avg:.1f}, max: {lag.max:.1f}' log.debug( 'Fps is %s. Total num frames: %d. Throughput: %s. Samples: %d. Policy #0 lag: (%s)', fps_str, total_env_steps, samples_per_policy, sum(self.samples_collected), policy_lag_str, ) if 'reward' in self.policy_avg_stats: policy_reward_stats = [] for policy_id in range(self.cfg.num_policies): reward_stats = self.policy_avg_stats['reward'][policy_id] if len(reward_stats) > 0: policy_reward_stats.append( (policy_id, f'{np.mean(reward_stats):.3f}')) log.debug('Avg episode reward: %r', policy_reward_stats)
def load_from_checkpoint(cfg): filename = cfg_file(cfg) if not os.path.isfile(filename): raise Exception( f'Could not load saved parameters for experiment {cfg.experiment}') with open(filename, 'r') as json_file: json_params = json.load(json_file) log.warning('Loading existing experiment configuration from %s', filename) loaded_cfg = AttrDict(json_params) # override the parameters in config file with values passed from command line for key, value in cfg.cli_args.items(): if key in loaded_cfg and loaded_cfg[key] != value: log.debug( 'Overriding arg %r with value %r passed from command line', key, value) loaded_cfg[key] = value # incorporate extra CLI parameters that were not present in JSON file for key, value in vars(cfg).items(): if key not in loaded_cfg: log.debug( 'Adding new argument %r=%r that is not in the saved config file!', key, value) loaded_cfg[key] = value return loaded_cfg
def maybe_load_from_checkpoint(cfg): filename = cfg_file(cfg) if not os.path.isfile(filename): log.warning( 'Saved parameter configuration for experiment %s not found!', cfg.experiment) log.warning('Starting experiment from scratch!') return AttrDict(vars(cfg)) return load_from_checkpoint(cfg)
def forward_tail(self, core_output, with_action_distribution=False): values = self.critic_linear(core_output) action_distribution_params, action_distribution = self.action_parameterization(core_output) # for non-trivial action spaces it is faster to do these together actions, log_prob_actions = sample_actions_log_probs(action_distribution) result = AttrDict(dict( actions=actions, action_logits=action_distribution_params, # perhaps `action_logits` is not the best name here since we now support continuous actions log_prob_actions=log_prob_actions, values=values, )) if with_action_distribution: result.action_distribution = action_distribution return result
def init(self): """ Actually instantiate the env instances. Also creates ActorState objects that hold the state of individual actors in (potentially) multi-agent envs. """ for env_i in range(self.num_envs): vector_idx = self.split_idx * self.num_envs + env_i # global env id within the entire system env_id = self.worker_idx * self.cfg.num_envs_per_worker + vector_idx env_config = AttrDict( worker_index=self.worker_idx, vector_index=vector_idx, env_id=env_id, ) # log.info('Creating env %r... %d-%d-%d', env_config, self.worker_idx, self.split_idx, env_i) env = make_env_func(self.cfg, env_config=env_config) env.seed(env_id) self.envs.append(env) actor_states_env, episode_rewards_env = [], [] for agent_idx in range(self.num_agents): agent_traj_tensors = self.traj_tensors.index( (env_i, agent_idx)) actor_state = ActorState( self.cfg, env, self.worker_idx, self.split_idx, env_i, agent_idx, agent_traj_tensors, self.num_traj_buffers, self.policy_outputs, self.policy_output_tensors[env_i, agent_idx], self.pbt_reward_shaping, self.policy_mgr, ) actor_states_env.append(actor_state) episode_rewards_env.append(0.0) self.actor_states.append(actor_states_env) self.episode_rewards.append(episode_rewards_env)
def test_env_performance(make_env, env_type, verbose=False): t = Timing() with t.timeit('init'): env = make_env(AttrDict({'worker_index': 0, 'vector_index': 0})) total_num_frames, frames = 10000, 0 with t.timeit('first_reset'): env.reset() t.reset = t.step = 1e-9 num_resets = 0 with t.timeit('experience'): while frames < total_num_frames: done = False start_reset = time.time() env.reset() t.reset += time.time() - start_reset num_resets += 1 while not done and frames < total_num_frames: start_step = time.time() if verbose: env.render() time.sleep(1.0 / 40) obs, rew, done, info = env.step(env.action_space.sample()) if verbose: log.info('Received reward %.3f', rew) t.step += time.time() - start_step frames += num_env_steps([info]) fps = total_num_frames / t.experience log.debug('%s performance:', env_type) log.debug('Took %.3f sec to collect %d frames on one CPU, %.1f FPS', t.experience, total_num_frames, fps) log.debug('Avg. reset time %.3f s', t.reset / num_resets) log.debug('Timing: %s', t) env.close()
def doom_multiagent(make_multi_env, worker_index, num_steps=1000): env_config = AttrDict({ 'worker_index': worker_index, 'vector_index': 0, 'safe_init': False }) multi_env = make_multi_env(env_config) obs = multi_env.reset() visualize = False start = time.time() for i in range(num_steps): actions = [multi_env.action_space.sample()] * len(obs) obs, rew, dones, infos = multi_env.step(actions) if visualize: multi_env.render() if i % 100 == 0 or any(dones): log.info('Rew %r done %r info %r', rew, dones, infos) if all(dones): multi_env.reset() took = time.time() - start log.info('Took %.3f seconds for %d steps', took, num_steps) log.info('Server steps per second: %.1f', num_steps / took) log.info('Observations fps: %.1f', num_steps * multi_env.num_agents / took) log.info( 'Environment fps: %.1f', num_steps * multi_env.num_agents * multi_env.skip_frames / took) multi_env.close()
def enjoy(cfg, max_num_frames=1e9): cfg = load_from_checkpoint(cfg) render_action_repeat = cfg.render_action_repeat if cfg.render_action_repeat is not None else cfg.env_frameskip if render_action_repeat is None: log.warning('Not using action repeat!') render_action_repeat = 1 log.debug('Using action repeat %d during evaluation', render_action_repeat) cfg.env_frameskip = 1 # for evaluation cfg.num_envs = 1 def make_env_func(env_config): return create_env(cfg.env, cfg=cfg, env_config=env_config) env = make_env_func(AttrDict({'worker_index': 0, 'vector_index': 0})) # env.seed(0) is_multiagent = is_multiagent_env(env) if not is_multiagent: env = MultiAgentWrapper(env) if hasattr(env.unwrapped, 'reset_on_init'): # reset call ruins the demo recording for VizDoom env.unwrapped.reset_on_init = False actor_critic = create_actor_critic(cfg, env.observation_space, env.action_space) device = torch.device('cpu' if cfg.device == 'cpu' else 'cuda') actor_critic.model_to_device(device) policy_id = cfg.policy_index checkpoints = LearnerWorker.get_checkpoints( LearnerWorker.checkpoint_dir(cfg, policy_id)) checkpoint_dict = LearnerWorker.load_checkpoint(checkpoints, device) actor_critic.load_state_dict(checkpoint_dict['model']) episode_rewards = [deque([], maxlen=100) for _ in range(env.num_agents)] true_rewards = [deque([], maxlen=100) for _ in range(env.num_agents)] num_frames = 0 last_render_start = time.time() def max_frames_reached(frames): return max_num_frames is not None and frames > max_num_frames obs = env.reset() rnn_states = torch.zeros( [env.num_agents, get_hidden_size(cfg)], dtype=torch.float32, device=device) episode_reward = np.zeros(env.num_agents) finished_episode = [False] * env.num_agents with torch.no_grad(): while not max_frames_reached(num_frames): obs_torch = AttrDict(transform_dict_observations(obs)) for key, x in obs_torch.items(): obs_torch[key] = torch.from_numpy(x).to(device).float() policy_outputs = actor_critic(obs_torch, rnn_states, with_action_distribution=True) # sample actions from the distribution by default actions = policy_outputs.actions action_distribution = policy_outputs.action_distribution if isinstance(action_distribution, ContinuousActionDistribution): if not cfg.continuous_actions_sample: # TODO: add similar option for discrete actions actions = action_distribution.means actions = actions.cpu().numpy() rnn_states = policy_outputs.rnn_states for _ in range(render_action_repeat): if not cfg.no_render: target_delay = 1.0 / cfg.fps if cfg.fps > 0 else 0 current_delay = time.time() - last_render_start time_wait = target_delay - current_delay if time_wait > 0: # log.info('Wait time %.3f', time_wait) time.sleep(time_wait) last_render_start = time.time() env.render() obs, rew, done, infos = env.step(actions) episode_reward += rew num_frames += 1 for agent_i, done_flag in enumerate(done): if done_flag: finished_episode[agent_i] = True episode_rewards[agent_i].append( episode_reward[agent_i]) true_rewards[agent_i].append(infos[agent_i].get( 'true_reward', episode_reward[agent_i])) log.info( 'Episode finished for agent %d at %d frames. Reward: %.3f, true_reward: %.3f', agent_i, num_frames, episode_reward[agent_i], true_rewards[agent_i][-1]) rnn_states[agent_i] = torch.zeros( [get_hidden_size(cfg)], dtype=torch.float32, device=device) episode_reward[agent_i] = 0 # if episode terminated synchronously for all agents, pause a bit before starting a new one if all(done): if not cfg.no_render: env.render() time.sleep(0.05) if all(finished_episode): finished_episode = [False] * env.num_agents avg_episode_rewards_str, avg_true_reward_str = '', '' for agent_i in range(env.num_agents): avg_rew = np.mean(episode_rewards[agent_i]) avg_true_rew = np.mean(true_rewards[agent_i]) if not np.isnan(avg_rew): if avg_episode_rewards_str: avg_episode_rewards_str += ', ' avg_episode_rewards_str += f'#{agent_i}: {avg_rew:.3f}' if not np.isnan(avg_true_rew): if avg_true_reward_str: avg_true_reward_str += ', ' avg_true_reward_str += f'#{agent_i}: {avg_true_rew:.3f}' log.info('Avg episode rewards: %s, true rewards: %s', avg_episode_rewards_str, avg_true_reward_str) log.info( 'Avg episode reward: %.3f, avg true_reward: %.3f', np.mean([ np.mean(episode_rewards[i]) for i in range(env.num_agents) ]), np.mean([ np.mean(true_rewards[i]) for i in range(env.num_agents) ])) # VizDoom multiplayer stuff # for player in [1, 2, 3, 4, 5, 6, 7, 8]: # key = f'PLAYER{player}_FRAGCOUNT' # if key in infos[0]: # log.debug('Score for player %d: %r', player, infos[0][key]) env.close() return ExperimentStatus.SUCCESS, np.mean(episode_rewards)
def sample(self, proc_idx): # workers should ignore Ctrl+C because the termination is handled in the event loop by a special msg signal.signal(signal.SIGINT, signal.SIG_IGN) if self.cfg.sampler_worker_gpus: set_gpus_for_process( proc_idx, num_gpus_per_process=1, process_type='sampler_proc', gpu_mask=self.cfg.sampler_worker_gpus, ) timing = Timing() from threadpoolctl import threadpool_limits with threadpool_limits(limits=1, user_api=None): if self.cfg.set_workers_cpu_affinity: set_process_cpu_affinity(proc_idx, self.cfg.num_workers) initial_cpu_affinity = psutil.Process().cpu_affinity( ) if platform != 'darwin' else None psutil.Process().nice(10) with timing.timeit('env_init'): envs = [] env_key = ['env' for _ in range(self.cfg.num_envs_per_worker)] for env_idx in range(self.cfg.num_envs_per_worker): global_env_id = proc_idx * self.cfg.num_envs_per_worker + env_idx env_config = AttrDict(worker_index=proc_idx, vector_index=env_idx, env_id=global_env_id) env = make_env_func(cfg=self.cfg, env_config=env_config) log.debug( 'CPU affinity after create_env: %r', psutil.Process().cpu_affinity() if platform != 'darwin' else 'MacOS - None') env.seed(global_env_id) envs.append(env) # this is to track the performance for individual DMLab levels if hasattr(env.unwrapped, 'level_name'): env_key[env_idx] = env.unwrapped.level_name episode_length = [0 for _ in envs] episode_lengths = [deque([], maxlen=20) for _ in envs] # sample a lot of random actions once, otherwise it is pretty slow in Python total_random_actions = 500 actions = [[ env.action_space.sample() for _ in range(env.num_agents) ] for _ in range(total_random_actions)] action_i = 0 try: with timing.timeit('first_reset'): for env_idx, env in enumerate(envs): env.reset() log.info('Process %d finished resetting %d/%d envs', proc_idx, env_idx + 1, len(envs)) self.report_queue.put( dict(proc_idx=proc_idx, finished_reset=True)) self.start_event.wait() with timing.timeit('work'): last_report = last_report_frames = total_env_frames = 0 while not self.terminate.value and total_env_frames < self.cfg.sample_env_frames_per_worker: for env_idx, env in enumerate(envs): with timing.add_time(f'{env_key[env_idx]}.step'): obs, rewards, dones, infos = env.step( actions[action_i]) action_i = (action_i + 1) % total_random_actions num_frames = sum( [info.get('num_frames', 1) for info in infos]) total_env_frames += num_frames episode_length[env_idx] += num_frames if all(dones): episode_lengths[env_idx].append( episode_length[env_idx] / env.num_agents) episode_length[env_idx] = 0 with timing.add_time('report'): now = time.time() if now - last_report > self.report_every_sec: last_report = now frames_since_last_report = total_env_frames - last_report_frames last_report_frames = total_env_frames self.report_queue.put( dict(proc_idx=proc_idx, env_frames=frames_since_last_report)) if proc_idx == 0: log.debug('Memory usage: %.4f Mb', memory_consumption_mb()) # Extra check to make sure cpu affinity is preserved throughout the execution. # I observed weird effect when some environments tried to alter affinity of the current process, leading # to decreased performance. # This can be caused by some interactions between deep learning libs, OpenCV, MKL, OpenMP, etc. # At least user should know about it if this is happening. cpu_affinity = psutil.Process().cpu_affinity( ) if platform != 'darwin' else None assert initial_cpu_affinity == cpu_affinity, \ f'Worker CPU affinity was changed from {initial_cpu_affinity} to {cpu_affinity}!' \ f'This can significantly affect performance!' except: log.exception('Unknown exception') log.error('Unknown exception in worker %d, terminating...', proc_idx) self.report_queue.put(dict(proc_idx=proc_idx, crash=True)) time.sleep(proc_idx * 0.01 + 0.01) log.info('Process %d finished sampling. Timing: %s', proc_idx, timing) for env_idx, env in enumerate(envs): if len(episode_lengths[env_idx]) > 0: log.warning('Level %s avg episode len %d', env_key[env_idx], np.mean(episode_lengths[env_idx])) for env in envs: env.close()
def sample(self, proc_idx): # workers should ignore Ctrl+C because the termination is handled in the event loop by a special msg signal.signal(signal.SIGINT, signal.SIG_IGN) timing = Timing() psutil.Process().nice(10) num_envs = len(DMLAB30_LEVELS_THAT_USE_LEVEL_CACHE) assert self.cfg.num_workers % num_envs == 0, f'should have an integer number of workers per env, e.g. {1 * num_envs}, {2 * num_envs}, etc...' assert self.cfg.num_envs_per_worker == 1, 'use populate_cache with 1 env per worker' with timing.timeit('env_init'): env_key = 'env' env_desired_num_levels = 0 global_env_id = proc_idx * self.cfg.num_envs_per_worker env_config = AttrDict(worker_index=proc_idx, vector_index=0, env_id=global_env_id) env = create_env(self.cfg.env, cfg=self.cfg, env_config=env_config) env.seed(global_env_id) # this is to track the performance for individual DMLab levels if hasattr(env.unwrapped, 'level_name'): env_key = env.unwrapped.level_name env_level = env.unwrapped.level approx_num_episodes_per_1b_frames = DMLAB30_APPROX_NUM_EPISODES_PER_BILLION_FRAMES[env_key] num_billions = DESIRED_TRAINING_LENGTH / int(1e9) num_workers_for_env = self.cfg.num_workers // num_envs env_desired_num_levels = int((approx_num_episodes_per_1b_frames * num_billions) / num_workers_for_env) env_num_levels_generated = len(dmlab_level_cache.DMLAB_GLOBAL_LEVEL_CACHE[0].all_seeds[env_level]) // num_workers_for_env log.warning('Worker %d (env %s) generated %d/%d levels!', proc_idx, env_key, env_num_levels_generated, env_desired_num_levels) time.sleep(4) env.reset() env_uses_level_cache = env.unwrapped.env_uses_level_cache self.report_queue.put(dict(proc_idx=proc_idx, finished_reset=True)) self.start_event.wait() try: with timing.timeit('work'): last_report = last_report_frames = total_env_frames = 0 while not self.terminate.value and total_env_frames < self.cfg.sample_env_frames_per_worker: action = env.action_space.sample() with timing.add_time(f'{env_key}.step'): env.step(action) total_env_frames += 1 with timing.add_time(f'{env_key}.reset'): env.reset() env_num_levels_generated += 1 log.debug('Env %s done %d/%d resets', env_key, env_num_levels_generated, env_desired_num_levels) if env_num_levels_generated >= env_desired_num_levels: log.debug('%s finished %d/%d resets, sleeping...', env_key, env_num_levels_generated, env_desired_num_levels) time.sleep(30) # free up CPU time for other envs # if env does not use level cache, there is no need to run it # let other workers proceed if not env_uses_level_cache: log.debug('Env %s does not require cache, sleeping...', env_key) time.sleep(200) with timing.add_time('report'): now = time.time() if now - last_report > self.report_every_sec: last_report = now frames_since_last_report = total_env_frames - last_report_frames last_report_frames = total_env_frames self.report_queue.put(dict(proc_idx=proc_idx, env_frames=frames_since_last_report)) if get_free_disk_space_mb(self.cfg) < 3 * 1024: log.error('Not enough disk space! %d', get_free_disk_space_mb(self.cfg)) time.sleep(200) except: log.exception('Unknown exception') log.error('Unknown exception in worker %d, terminating...', proc_idx) self.report_queue.put(dict(proc_idx=proc_idx, crash=True)) time.sleep(proc_idx * 0.1 + 0.1) log.info('Process %d finished sampling. Timing: %s', proc_idx, timing) env.close()
def multi_agent_match(policy_indices, max_num_episodes=int(1e9), max_num_frames=1e10): log.debug('Starting eval process with policies %r', policy_indices) for i, rival in enumerate(RIVALS): rival.policy_index = policy_indices[i] curr_dir = os.path.dirname(os.path.abspath(__file__)) evaluation_filename = join(curr_dir, f'eval_{"vs".join([str(pi) for pi in policy_indices])}.txt') with open(evaluation_filename, 'w') as fobj: fobj.write('start\n') common_config = RIVALS[0].cfg render_action_repeat = common_config.render_action_repeat if common_config.render_action_repeat is not None else common_config.env_frameskip if render_action_repeat is None: log.warning('Not using action repeat!') render_action_repeat = 1 log.debug('Using action repeat %d during evaluation', render_action_repeat) common_config.env_frameskip = 1 # for evaluation common_config.num_envs = 1 common_config.timelimit = 4.0 # for faster evaluation def make_env_func(env_config): return create_env(ENV_NAME, cfg=common_config, env_config=env_config) env = make_env_func(AttrDict({'worker_index': 0, 'vector_index': 0})) env.seed(0) is_multiagent = is_multiagent_env(env) if not is_multiagent: env = MultiAgentWrapper(env) else: assert env.num_agents == len(RIVALS) device = torch.device('cuda') for rival in RIVALS: rival.actor_critic = create_actor_critic(rival.cfg, env.observation_space, env.action_space) rival.actor_critic.model_to_device(device) policy_id = rival.policy_index checkpoints = LearnerWorker.get_checkpoints(LearnerWorker.checkpoint_dir(rival.cfg, policy_id)) checkpoint_dict = LearnerWorker.load_checkpoint(checkpoints, device) rival.actor_critic.load_state_dict(checkpoint_dict['model']) episode_rewards = [] num_frames = 0 last_render_start = time.time() def max_frames_reached(frames): return max_num_frames is not None and frames > max_num_frames wins = [0 for _ in RIVALS] ties = 0 frag_differences = [] with torch.no_grad(): for _ in range(max_num_episodes): obs = env.reset() obs_dict_torch = dict() done = [False] * len(obs) for rival in RIVALS: rival.rnn_states = torch.zeros([1, rival.cfg.hidden_size], dtype=torch.float32, device=device) episode_reward = 0 prev_frame = time.time() while True: actions = [] for i, obs_dict in enumerate(obs): for key, x in obs_dict.items(): obs_dict_torch[key] = torch.from_numpy(x).to(device).float().view(1, *x.shape) rival = RIVALS[i] policy_outputs = rival.actor_critic(obs_dict_torch, rival.rnn_states) rival.rnn_states = policy_outputs.rnn_states actions.append(policy_outputs.actions[0].cpu().numpy()) for _ in range(render_action_repeat): if not NO_RENDER: target_delay = 1.0 / FPS if FPS > 0 else 0 current_delay = time.time() - last_render_start time_wait = target_delay - current_delay if time_wait > 0: # log.info('Wait time %.3f', time_wait) time.sleep(time_wait) last_render_start = time.time() env.render() obs, rew, done, infos = env.step(actions) if all(done): log.debug('Finished episode!') frag_diff = infos[0]['PLAYER1_FRAGCOUNT'] - infos[0]['PLAYER2_FRAGCOUNT'] if frag_diff > 0: wins[0] += 1 elif frag_diff < 0: wins[1] += 1 else: ties += 1 frag_differences.append(frag_diff) avg_frag_diff = np.mean(frag_differences) report = f'wins: {wins}, ties: {ties}, avg_frag_diff: {avg_frag_diff}' with open(evaluation_filename, 'a') as fobj: fobj.write(report + '\n') # log.info('%d:%d', infos[0]['PLAYER1_FRAGCOUNT'], infos[0]['PLAYER2_FRAGCOUNT']) episode_reward += np.mean(rew) num_frames += 1 if num_frames % 100 == 0: log.debug('%.1f', render_action_repeat / (time.time() - prev_frame)) prev_frame = time.time() if all(done): log.info('Episode finished at %d frames', num_frames) break if all(done) or max_frames_reached(num_frames): break if not NO_RENDER: env.render() time.sleep(0.01) episode_rewards.append(episode_reward) last_episodes = episode_rewards[-100:] avg_reward = sum(last_episodes) / len(last_episodes) log.info( 'Episode reward: %f, avg reward for %d episodes: %f', episode_reward, len(last_episodes), avg_reward, ) if max_frames_reached(num_frames): break env.close()