def _init_config(self, config, args):
if args.build_port is not None:
config['base_config']['build_port'] = args.build_port
if args.nn is not None:
config['base_config']['nn'] = args.nn
if args.agents is not None:
config['base_config']['n_agents'] = args.agents
self.base_config = config['base_config']
register_response = self._stub.register_to_learner()
(model_abs_dir, _id, reset_config, model_config,
sac_config) = register_response
self._logger.info(f'Assigned to id {_id}')
config['reset_config'] = self.reset_config = reset_config
config['model_config'] = self.model_config = model_config
config['sac_config'] = sac_config
self.sac_config = config['sac_config']
self.model_abs_dir = model_abs_dir
# Set logger file if available
if self.logger_in_file:
logger_file = Path(model_abs_dir).joinpath(f'actor-{_id}.log')
config_helper.set_logger(logger_file)
self._logger.info(f'Set to logger {logger_file}')
config_helper.display_config(config, self._logger)
def __init__(self, logger_in_file: bool, model_abs_dir: str, use_rnn: bool,
burn_in_step: int, n_step: int, batch_size: int,
episode_buffer: SharedMemoryManager,
episode_size_array: mp.Array,
batch_buffer: SharedMemoryManager):
self.use_rnn = use_rnn
self.burn_in_step = burn_in_step
self.n_step = n_step
self.batch_size = batch_size
self._episode_buffer = episode_buffer
self._episode_size_array = episode_size_array
self._batch_buffer = batch_buffer
# Since no set_logger() in main.py
config_helper.set_logger(
Path(model_abs_dir).
joinpath(f'learner_trainer_batch_generator_{os.getpid()}.log'
) if logger_in_file else None)
self._logger = logging.getLogger(
f'ds.learner.trainer.batch_generator_{os.getpid()}')
self._logger.info(f'BatchGenerator {os.getpid()} initialized')
episode_buffer.init_logger(self._logger)
batch_buffer.init_logger(self._logger)
self.run()
def start_gym_process(env_name, render, conn):
try:
from algorithm import config_helper
config_helper.set_logger()
except:
pass
logger = logging.getLogger('GymWrapper.Process')
logger.info(f'Process {os.getpid()} created')
env = gym.make(env_name)
if render:
env.render()
try:
while True:
cmd, data = conn.recv()
if cmd == RESET:
obs = env.reset()
conn.send(obs)
elif cmd == CLOSE:
env.close()
break
elif cmd == STEP:
conn.send(env.step(data))
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(e)
logger.warning(f'Process {os.getpid()} exits')
def _init_config(self, root_dir, config_dir, args):
default_config_file_path = Path(__file__).resolve().parent.joinpath(
'default_config.yaml')
config_abs_dir = Path(root_dir).joinpath(config_dir)
config_abs_path = config_abs_dir.joinpath('config_ds.yaml')
config = config_helper.initialize_config_from_yaml(
default_config_file_path,
config_abs_path,
args.config,
is_evolver=True)
if args.evolver_port is not None:
config['net_config']['evolver_port'] = args.evolver_port
if args.name is not None:
config['base_config']['name'] = args.name
config['base_config']['name'] = config_helper.generate_base_name(
config['base_config']['name'], 'ds')
model_abs_dir = Path(root_dir).joinpath('models',
config['base_config']['scene'],
config['base_config']['name'])
model_abs_dir.mkdir(parents=True, exist_ok=True)
if args.logger_in_file:
config_helper.set_logger(
Path(model_abs_dir).joinpath('evolver.log'))
config_helper.save_config(config, model_abs_dir, 'config_ds.yaml')
config_helper.display_config(config, self._logger)
self.config = config
def _init_config(self, root_dir, config_dir, args):
config_abs_dir = Path(root_dir).joinpath(config_dir)
config_abs_path = config_abs_dir.joinpath('config.yaml')
default_config_abs_path = Path(__file__).resolve().parent.joinpath(
'default_config.yaml')
# Merge default_config.yaml and custom config.yaml
config = config_helper.initialize_config_from_yaml(
default_config_abs_path, config_abs_path, args.config)
# Initialize config from command line arguments
self.train_mode = not args.run
self.render = args.render
self.run_in_editor = args.editor
self.additional_args = args.additional_args
self.disable_sample = args.disable_sample
self.alway_use_env_nn = args.use_env_nn
self.device = args.device
self.last_ckpt = args.ckpt
if args.name is not None:
config['base_config']['name'] = args.name
if args.port is not None:
config['base_config']['port'] = args.port
if args.nn is not None:
config['base_config']['nn'] = args.nn
if args.agents is not None:
config['base_config']['n_agents'] = args.agents
if args.max_iter is not None:
config['base_config']['max_iter'] = args.max_iter
config['base_config']['name'] = config_helper.generate_base_name(
config['base_config']['name'])
# The absolute directory of a specific training
model_abs_dir = Path(root_dir).joinpath('models',
config['base_config']['scene'],
config['base_config']['name'])
model_abs_dir.mkdir(parents=True, exist_ok=True)
self.model_abs_dir = model_abs_dir
if args.logger_in_file:
config_helper.set_logger(Path(model_abs_dir).joinpath(f'log.log'))
if self.train_mode:
config_helper.save_config(config, model_abs_dir, 'config.yaml')
config_helper.display_config(config, self._logger)
self.base_config = config['base_config']
self.reset_config = config['reset_config']
self.model_config = config['model_config']
self.replay_config = config['replay_config']
self.sac_config = config['sac_config']
return config_abs_dir
def _init_config(self, config_path, args):
config_file_path = f'{config_path}/config_ds.yaml'
config = config_helper.initialize_config_from_yaml(
f'{Path(__file__).resolve().parent}/default_config.yaml',
config_file_path, args.config)
self.logger = config_helper.set_logger('ds.replay', args.logger_file)
config_helper.display_config(config, self.logger)
return (config['base_config'], config['net_config'],
config['replay_config'], config['sac_config'])
def __init__(self, logger_in_file: bool, model_abs_dir: str,
learner_host: str, learner_port: int,
episode_buffer: SharedMemoryManager,
episode_size_array: mp.Array):
self._episode_buffer = episode_buffer
self._episode_size_array = episode_size_array
config_helper.set_logger(
Path(model_abs_dir).
joinpath(f'actor_episode_sender_{os.getpid()}.log'
) if logger_in_file else None)
self._logger = logging.getLogger(
f'ds.actor.episode_sender_{os.getpid()}')
self._stub = StubEpisodeSenderController(learner_host, learner_port)
self._logger.info(f'EpisodeSender {os.getpid()} initialized')
episode_buffer.init_logger(self._logger)
self._run()
def _init_constant_config(self, config_path, args):
config_file_path = f'{config_path}/config_ds.yaml'
config = config_helper.initialize_config_from_yaml(
f'{Path(__file__).resolve().parent}/default_config.yaml',
config_file_path, args.config)
self.config_file_path = config_file_path
# Initialize config from command line arguments
self.train_mode = not args.run
self.run_in_editor = args.editor
self.config_cat = args.config
self.logger = config_helper.set_logger('ds.actor', args.logger_file)
return config['net_config']
def _init_config(self, config_path, args):
config_file_path = f'{config_path}/config.yaml'
# Merge default_config.yaml and custom config.yaml
config = config_helper.initialize_config_from_yaml(f'{Path(__file__).resolve().parent}/default_config.yaml',
config_file_path,
args.config)
# Initialize config from command line arguments
self.train_mode = not args.run
self.last_ckpt = args.ckpt
self.render = args.render
self.run_in_editor = args.editor
if args.name is not None:
config['base_config']['name'] = args.name
if args.port is not None:
config['base_config']['port'] = args.port
if args.sac is not None:
config['base_config']['sac'] = args.sac
if args.agents is not None:
config['base_config']['n_agents'] = args.agents
# Replace {time} from current time and random letters
rand = ''.join(random.sample(string.ascii_letters, 4))
config['base_config']['name'] = config['base_config']['name'].replace('{time}', self._now + rand)
model_root_path = f'models/{config["base_config"]["scene"]}/{config["base_config"]["name"]}'
logger_file = f'{model_root_path}/{args.logger_file}' if args.logger_file is not None else None
self.logger = config_helper.set_logger('sac', logger_file)
if self.train_mode:
config_helper.save_config(config, model_root_path, 'config.yaml')
config_helper.display_config(config, self.logger)
return (config['base_config'],
config['reset_config'],
config['replay_config'],
config['sac_config'],
model_root_path)
def _init_config(self, config_path, args):
config_file_path = f'{config_path}/config_ds.yaml'
config = config_helper.initialize_config_from_yaml(
f'{Path(__file__).resolve().parent}/default_config.yaml',
config_file_path, args.config)
# Initialize config from command line arguments
self.train_mode = not args.run
self.last_ckpt = args.ckpt
self.render = args.render
self.run_in_editor = args.editor
if args.name is not None:
config['base_config']['name'] = args.name
if args.build_port is not None:
config['base_config']['build_port'] = args.build_port
if args.sac is not None:
config['base_config']['sac'] = args.sac
if args.agents is not None:
config['base_config']['n_agents'] = args.agents
config['base_config']['name'] = config['base_config']['name'].replace(
'{time}', self._now)
model_root_path = f'models/ds/{config["base_config"]["scene"]}/{config["base_config"]["name"]}'
logger_file = f'{model_root_path}/{args.logger_file}' if args.logger_file is not None else None
self.logger = config_helper.set_logger('ds.learner', logger_file)
if self.train_mode:
config_helper.save_config(config, model_root_path, 'config.yaml')
config_helper.display_config(config, self.logger)
return (config['base_config'], config['net_config'],
config['reset_config'], config['replay_config'],
config['sac_config'], model_root_path)
def __init__(self, all_variables_buffer: SharedMemoryManager,
episode_buffer: SharedMemoryManager,
episode_size_array: mp.Array, cmd_pipe_server: Connection,
logger_in_file, obs_shapes, d_action_size, c_action_size,
model_abs_dir, model_spec, device, last_ckpt, config):
self._all_variables_buffer = all_variables_buffer
# Since no set_logger() in main.py
config_helper.set_logger(
Path(model_abs_dir).joinpath('learner_trainer.log'
) if logger_in_file else None)
self._logger = logging.getLogger('ds.learner.trainer')
all_variables_buffer.init_logger(self._logger)
self.base_config = config['base_config']
custom_nn_model = importlib.util.module_from_spec(model_spec)
model_spec.loader.exec_module(custom_nn_model)
self.sac_lock = RLock(1)
self.sac = SAC_DS_Base(obs_shapes=obs_shapes,
d_action_size=d_action_size,
c_action_size=c_action_size,
model_abs_dir=model_abs_dir,
model=custom_nn_model,
model_config=config['model_config'],
device=device,
last_ckpt=last_ckpt,
**config['sac_config'])
self._logger.info('SAC started')
batch_size = config['sac_config']['batch_size']
bn = self.sac.burn_in_step + self.sac.n_step
batch_shapes = [
[(batch_size, bn, *o) for o in obs_shapes],
(batch_size, bn, d_action_size + c_action_size), (batch_size, bn),
[(batch_size, *o) for o in obs_shapes], (batch_size, bn),
(batch_size, bn),
(batch_size,
*self.sac.rnn_state_shape) if self.sac.use_rnn else None
]
batch_dtypes = [[np.float32 for _ in obs_shapes], np.float32,
np.float32, [np.float32 for _ in obs_shapes], bool,
np.float32, np.float32 if self.sac.use_rnn else None]
self._batch_buffer = SharedMemoryManager(
self.base_config['batch_queue_size'],
logger=self._logger,
counter_get_shm_index_empty_log='Batch shm index is empty',
timer_get_shm_index_log='Get a batch shm index',
timer_get_data_log='Get a batch',
log_repeat=ELAPSED_REPEAT)
self._batch_buffer.init_from_shapes(batch_shapes, batch_dtypes)
for _ in range(self.base_config['batch_generator_process_num']):
mp.Process(target=BatchGenerator,
kwargs={
'logger_in_file': logger_in_file,
'model_abs_dir': model_abs_dir,
'use_rnn': self.sac.use_rnn,
'burn_in_step': self.sac.burn_in_step,
'n_step': self.sac.n_step,
'batch_size': batch_size,
'episode_buffer': episode_buffer,
'episode_size_array': episode_size_array,
'batch_buffer': self._batch_buffer
}).start()
threading.Thread(target=self._forever_run_cmd_pipe,
args=[cmd_pipe_server],
daemon=True).start()
self._update_sac_bak()
self.run_train()
obs_list = next_obs_list
step += 1
except Exception as e:
self._logger.error(e)
self._logger.error('Exiting...')
break
iteration += 1
self.env.close()
if __name__ == '__main__':
set_logger()
parser = argparse.ArgumentParser()
parser.add_argument('env')
parser.add_argument('--config', '-c', help='config file')
parser.add_argument('--run', action='store_true', help='inference mode')
parser.add_argument('--render', action='store_true', help='render')
parser.add_argument('--editor',
action='store_true',
help='running in Unity Editor')
parser.add_argument('--additional_args', help='additional args for Unity')
parser.add_argument('--port',
'-p',
type=int,
default=5005,
def __init__(self,
conn: multiprocessing.connection.Connection = None,
train_mode=True,
file_name=None,
worker_id=0,
base_port=5005,
no_graphics=True,
seed=None,
scene=None,
additional_args=None,
n_agents=1):
"""
Args:
train_mode: If in train mode, Unity will speed up
file_name: The executable path. The UnityEnvironment will run in editor if None
worker_id: Offset from base_port
base_port: The port that communicate to Unity. It will be set to 5004 automatically if in editor.
no_graphics: If Unity runs in no graphic mode. It must be set to False if Unity has camera sensor.
seed: Random seed
scene: The scene name
n_agents: The agents count
"""
self.scene = scene
self.n_agents = n_agents
seed = seed if seed is not None else np.random.randint(0, 65536)
additional_args = [] if additional_args is None else additional_args.split(
' ')
self.engine_configuration_channel = EngineConfigurationChannel()
self.environment_parameters_channel = EnvironmentParametersChannel()
self.environment_parameters_channel.set_float_parameter(
'env_copys', float(n_agents))
if conn:
try:
from algorithm import config_helper
config_helper.set_logger()
except:
pass
self._logger = logging.getLogger(
f'UnityWrapper.Process_{os.getpid()}')
else:
self._logger = logging.getLogger('UnityWrapper.Process')
self._env = UnityEnvironment(
file_name=file_name,
worker_id=worker_id,
base_port=base_port if file_name else None,
no_graphics=no_graphics and train_mode,
seed=seed,
additional_args=['--scene', scene] + additional_args,
side_channels=[
self.engine_configuration_channel,
self.environment_parameters_channel
])
self.engine_configuration_channel.set_configuration_parameters(
width=200 if train_mode else 1280,
height=200 if train_mode else 720,
quality_level=5,
time_scale=20 if train_mode else 1)
self._env.reset()
self.bahavior_name = list(self._env.behavior_specs)[0]
if conn:
try:
while True:
cmd, data = conn.recv()
if cmd == INIT:
conn.send(self.init())
elif cmd == RESET:
conn.send(self.reset(data))
elif cmd == STEP:
conn.send(self.step(*data))
elif cmd == CLOSE:
self.close()
except:
self._logger.error(traceback.format_exc())
