def _init_config(self, config, args): if args.build_port is not None: config['base_config']['build_port'] = args.build_port if args.nn is not None: config['base_config']['nn'] = args.nn if args.agents is not None: config['base_config']['n_agents'] = args.agents self.base_config = config['base_config'] register_response = self._stub.register_to_learner() (model_abs_dir, _id, reset_config, model_config, sac_config) = register_response self._logger.info(f'Assigned to id {_id}') config['reset_config'] = self.reset_config = reset_config config['model_config'] = self.model_config = model_config config['sac_config'] = sac_config self.sac_config = config['sac_config'] self.model_abs_dir = model_abs_dir # Set logger file if available if self.logger_in_file: logger_file = Path(model_abs_dir).joinpath(f'actor-{_id}.log') config_helper.set_logger(logger_file) self._logger.info(f'Set to logger {logger_file}') config_helper.display_config(config, self._logger)
def __init__(self, logger_in_file: bool, model_abs_dir: str, use_rnn: bool, burn_in_step: int, n_step: int, batch_size: int, episode_buffer: SharedMemoryManager, episode_size_array: mp.Array, batch_buffer: SharedMemoryManager): self.use_rnn = use_rnn self.burn_in_step = burn_in_step self.n_step = n_step self.batch_size = batch_size self._episode_buffer = episode_buffer self._episode_size_array = episode_size_array self._batch_buffer = batch_buffer # Since no set_logger() in main.py config_helper.set_logger( Path(model_abs_dir). joinpath(f'learner_trainer_batch_generator_{os.getpid()}.log' ) if logger_in_file else None) self._logger = logging.getLogger( f'ds.learner.trainer.batch_generator_{os.getpid()}') self._logger.info(f'BatchGenerator {os.getpid()} initialized') episode_buffer.init_logger(self._logger) batch_buffer.init_logger(self._logger) self.run()
def start_gym_process(env_name, render, conn): try: from algorithm import config_helper config_helper.set_logger() except: pass logger = logging.getLogger('GymWrapper.Process') logger.info(f'Process {os.getpid()} created') env = gym.make(env_name) if render: env.render() try: while True: cmd, data = conn.recv() if cmd == RESET: obs = env.reset() conn.send(obs) elif cmd == CLOSE: env.close() break elif cmd == STEP: conn.send(env.step(data)) except KeyboardInterrupt: pass except Exception as e: logger.error(e) logger.warning(f'Process {os.getpid()} exits')
def _init_config(self, root_dir, config_dir, args): default_config_file_path = Path(__file__).resolve().parent.joinpath( 'default_config.yaml') config_abs_dir = Path(root_dir).joinpath(config_dir) config_abs_path = config_abs_dir.joinpath('config_ds.yaml') config = config_helper.initialize_config_from_yaml( default_config_file_path, config_abs_path, args.config, is_evolver=True) if args.evolver_port is not None: config['net_config']['evolver_port'] = args.evolver_port if args.name is not None: config['base_config']['name'] = args.name config['base_config']['name'] = config_helper.generate_base_name( config['base_config']['name'], 'ds') model_abs_dir = Path(root_dir).joinpath('models', config['base_config']['scene'], config['base_config']['name']) model_abs_dir.mkdir(parents=True, exist_ok=True) if args.logger_in_file: config_helper.set_logger( Path(model_abs_dir).joinpath('evolver.log')) config_helper.save_config(config, model_abs_dir, 'config_ds.yaml') config_helper.display_config(config, self._logger) self.config = config
def _init_config(self, root_dir, config_dir, args): config_abs_dir = Path(root_dir).joinpath(config_dir) config_abs_path = config_abs_dir.joinpath('config.yaml') default_config_abs_path = Path(__file__).resolve().parent.joinpath( 'default_config.yaml') # Merge default_config.yaml and custom config.yaml config = config_helper.initialize_config_from_yaml( default_config_abs_path, config_abs_path, args.config) # Initialize config from command line arguments self.train_mode = not args.run self.render = args.render self.run_in_editor = args.editor self.additional_args = args.additional_args self.disable_sample = args.disable_sample self.alway_use_env_nn = args.use_env_nn self.device = args.device self.last_ckpt = args.ckpt if args.name is not None: config['base_config']['name'] = args.name if args.port is not None: config['base_config']['port'] = args.port if args.nn is not None: config['base_config']['nn'] = args.nn if args.agents is not None: config['base_config']['n_agents'] = args.agents if args.max_iter is not None: config['base_config']['max_iter'] = args.max_iter config['base_config']['name'] = config_helper.generate_base_name( config['base_config']['name']) # The absolute directory of a specific training model_abs_dir = Path(root_dir).joinpath('models', config['base_config']['scene'], config['base_config']['name']) model_abs_dir.mkdir(parents=True, exist_ok=True) self.model_abs_dir = model_abs_dir if args.logger_in_file: config_helper.set_logger(Path(model_abs_dir).joinpath(f'log.log')) if self.train_mode: config_helper.save_config(config, model_abs_dir, 'config.yaml') config_helper.display_config(config, self._logger) self.base_config = config['base_config'] self.reset_config = config['reset_config'] self.model_config = config['model_config'] self.replay_config = config['replay_config'] self.sac_config = config['sac_config'] return config_abs_dir
def _init_config(self, config_path, args): config_file_path = f'{config_path}/config_ds.yaml' config = config_helper.initialize_config_from_yaml( f'{Path(__file__).resolve().parent}/default_config.yaml', config_file_path, args.config) self.logger = config_helper.set_logger('ds.replay', args.logger_file) config_helper.display_config(config, self.logger) return (config['base_config'], config['net_config'], config['replay_config'], config['sac_config'])
def __init__(self, logger_in_file: bool, model_abs_dir: str, learner_host: str, learner_port: int, episode_buffer: SharedMemoryManager, episode_size_array: mp.Array): self._episode_buffer = episode_buffer self._episode_size_array = episode_size_array config_helper.set_logger( Path(model_abs_dir). joinpath(f'actor_episode_sender_{os.getpid()}.log' ) if logger_in_file else None) self._logger = logging.getLogger( f'ds.actor.episode_sender_{os.getpid()}') self._stub = StubEpisodeSenderController(learner_host, learner_port) self._logger.info(f'EpisodeSender {os.getpid()} initialized') episode_buffer.init_logger(self._logger) self._run()
def _init_constant_config(self, config_path, args): config_file_path = f'{config_path}/config_ds.yaml' config = config_helper.initialize_config_from_yaml( f'{Path(__file__).resolve().parent}/default_config.yaml', config_file_path, args.config) self.config_file_path = config_file_path # Initialize config from command line arguments self.train_mode = not args.run self.run_in_editor = args.editor self.config_cat = args.config self.logger = config_helper.set_logger('ds.actor', args.logger_file) return config['net_config']
def _init_config(self, config_path, args): config_file_path = f'{config_path}/config.yaml' # Merge default_config.yaml and custom config.yaml config = config_helper.initialize_config_from_yaml(f'{Path(__file__).resolve().parent}/default_config.yaml', config_file_path, args.config) # Initialize config from command line arguments self.train_mode = not args.run self.last_ckpt = args.ckpt self.render = args.render self.run_in_editor = args.editor if args.name is not None: config['base_config']['name'] = args.name if args.port is not None: config['base_config']['port'] = args.port if args.sac is not None: config['base_config']['sac'] = args.sac if args.agents is not None: config['base_config']['n_agents'] = args.agents # Replace {time} from current time and random letters rand = ''.join(random.sample(string.ascii_letters, 4)) config['base_config']['name'] = config['base_config']['name'].replace('{time}', self._now + rand) model_root_path = f'models/{config["base_config"]["scene"]}/{config["base_config"]["name"]}' logger_file = f'{model_root_path}/{args.logger_file}' if args.logger_file is not None else None self.logger = config_helper.set_logger('sac', logger_file) if self.train_mode: config_helper.save_config(config, model_root_path, 'config.yaml') config_helper.display_config(config, self.logger) return (config['base_config'], config['reset_config'], config['replay_config'], config['sac_config'], model_root_path)
def _init_config(self, config_path, args): config_file_path = f'{config_path}/config_ds.yaml' config = config_helper.initialize_config_from_yaml( f'{Path(__file__).resolve().parent}/default_config.yaml', config_file_path, args.config) # Initialize config from command line arguments self.train_mode = not args.run self.last_ckpt = args.ckpt self.render = args.render self.run_in_editor = args.editor if args.name is not None: config['base_config']['name'] = args.name if args.build_port is not None: config['base_config']['build_port'] = args.build_port if args.sac is not None: config['base_config']['sac'] = args.sac if args.agents is not None: config['base_config']['n_agents'] = args.agents config['base_config']['name'] = config['base_config']['name'].replace( '{time}', self._now) model_root_path = f'models/ds/{config["base_config"]["scene"]}/{config["base_config"]["name"]}' logger_file = f'{model_root_path}/{args.logger_file}' if args.logger_file is not None else None self.logger = config_helper.set_logger('ds.learner', logger_file) if self.train_mode: config_helper.save_config(config, model_root_path, 'config.yaml') config_helper.display_config(config, self.logger) return (config['base_config'], config['net_config'], config['reset_config'], config['replay_config'], config['sac_config'], model_root_path)
def __init__(self, all_variables_buffer: SharedMemoryManager, episode_buffer: SharedMemoryManager, episode_size_array: mp.Array, cmd_pipe_server: Connection, logger_in_file, obs_shapes, d_action_size, c_action_size, model_abs_dir, model_spec, device, last_ckpt, config): self._all_variables_buffer = all_variables_buffer # Since no set_logger() in main.py config_helper.set_logger( Path(model_abs_dir).joinpath('learner_trainer.log' ) if logger_in_file else None) self._logger = logging.getLogger('ds.learner.trainer') all_variables_buffer.init_logger(self._logger) self.base_config = config['base_config'] custom_nn_model = importlib.util.module_from_spec(model_spec) model_spec.loader.exec_module(custom_nn_model) self.sac_lock = RLock(1) self.sac = SAC_DS_Base(obs_shapes=obs_shapes, d_action_size=d_action_size, c_action_size=c_action_size, model_abs_dir=model_abs_dir, model=custom_nn_model, model_config=config['model_config'], device=device, last_ckpt=last_ckpt, **config['sac_config']) self._logger.info('SAC started') batch_size = config['sac_config']['batch_size'] bn = self.sac.burn_in_step + self.sac.n_step batch_shapes = [ [(batch_size, bn, *o) for o in obs_shapes], (batch_size, bn, d_action_size + c_action_size), (batch_size, bn), [(batch_size, *o) for o in obs_shapes], (batch_size, bn), (batch_size, bn), (batch_size, *self.sac.rnn_state_shape) if self.sac.use_rnn else None ] batch_dtypes = [[np.float32 for _ in obs_shapes], np.float32, np.float32, [np.float32 for _ in obs_shapes], bool, np.float32, np.float32 if self.sac.use_rnn else None] self._batch_buffer = SharedMemoryManager( self.base_config['batch_queue_size'], logger=self._logger, counter_get_shm_index_empty_log='Batch shm index is empty', timer_get_shm_index_log='Get a batch shm index', timer_get_data_log='Get a batch', log_repeat=ELAPSED_REPEAT) self._batch_buffer.init_from_shapes(batch_shapes, batch_dtypes) for _ in range(self.base_config['batch_generator_process_num']): mp.Process(target=BatchGenerator, kwargs={ 'logger_in_file': logger_in_file, 'model_abs_dir': model_abs_dir, 'use_rnn': self.sac.use_rnn, 'burn_in_step': self.sac.burn_in_step, 'n_step': self.sac.n_step, 'batch_size': batch_size, 'episode_buffer': episode_buffer, 'episode_size_array': episode_size_array, 'batch_buffer': self._batch_buffer }).start() threading.Thread(target=self._forever_run_cmd_pipe, args=[cmd_pipe_server], daemon=True).start() self._update_sac_bak() self.run_train()
obs_list = next_obs_list step += 1 except Exception as e: self._logger.error(e) self._logger.error('Exiting...') break iteration += 1 self.env.close() if __name__ == '__main__': set_logger() parser = argparse.ArgumentParser() parser.add_argument('env') parser.add_argument('--config', '-c', help='config file') parser.add_argument('--run', action='store_true', help='inference mode') parser.add_argument('--render', action='store_true', help='render') parser.add_argument('--editor', action='store_true', help='running in Unity Editor') parser.add_argument('--additional_args', help='additional args for Unity') parser.add_argument('--port', '-p', type=int, default=5005,
def __init__(self, conn: multiprocessing.connection.Connection = None, train_mode=True, file_name=None, worker_id=0, base_port=5005, no_graphics=True, seed=None, scene=None, additional_args=None, n_agents=1): """ Args: train_mode: If in train mode, Unity will speed up file_name: The executable path. The UnityEnvironment will run in editor if None worker_id: Offset from base_port base_port: The port that communicate to Unity. It will be set to 5004 automatically if in editor. no_graphics: If Unity runs in no graphic mode. It must be set to False if Unity has camera sensor. seed: Random seed scene: The scene name n_agents: The agents count """ self.scene = scene self.n_agents = n_agents seed = seed if seed is not None else np.random.randint(0, 65536) additional_args = [] if additional_args is None else additional_args.split( ' ') self.engine_configuration_channel = EngineConfigurationChannel() self.environment_parameters_channel = EnvironmentParametersChannel() self.environment_parameters_channel.set_float_parameter( 'env_copys', float(n_agents)) if conn: try: from algorithm import config_helper config_helper.set_logger() except: pass self._logger = logging.getLogger( f'UnityWrapper.Process_{os.getpid()}') else: self._logger = logging.getLogger('UnityWrapper.Process') self._env = UnityEnvironment( file_name=file_name, worker_id=worker_id, base_port=base_port if file_name else None, no_graphics=no_graphics and train_mode, seed=seed, additional_args=['--scene', scene] + additional_args, side_channels=[ self.engine_configuration_channel, self.environment_parameters_channel ]) self.engine_configuration_channel.set_configuration_parameters( width=200 if train_mode else 1280, height=200 if train_mode else 720, quality_level=5, time_scale=20 if train_mode else 1) self._env.reset() self.bahavior_name = list(self._env.behavior_specs)[0] if conn: try: while True: cmd, data = conn.recv() if cmd == INIT: conn.send(self.init()) elif cmd == RESET: conn.send(self.reset(data)) elif cmd == STEP: conn.send(self.step(*data)) elif cmd == CLOSE: self.close() except: self._logger.error(traceback.format_exc())