def __init__(self, env_args: Config, model_args: Config, buffer_args: Config, train_args: Config): # print("89898989") self.env_args = env_args self.model_args = model_args self.buffer_args = buffer_args self.train_args = train_args self.use_GCN = False self.model_index = str(self.train_args.get('index')) self.all_learner_print = bool( self.train_args.get('all_learner_print', False)) if '-' not in self.train_args['name']: self.train_args['name'] += f'-{self.model_index}' if self.model_args['load'] is None: self.train_args['load_model_path'] = os.path.join( self.train_args['base_dir'], self.train_args['name']) else: if '/' in self.model_args['load'] or '\\' in self.model_args[ 'load']: # 所有训练进程都以该模型路径初始化,绝对路径 self.train_args['load_model_path'] = self.model_args['load'] elif '-' in self.model_args['load']: self.train_args['load_model_path'] = os.path.join( self.train_args['base_dir'], self.model_args['load']) # 指定了名称和序号,所有训练进程都以该模型路径初始化,相对路径 else: # 只写load的训练名称,不用带进程序号,会自动补 self.train_args['load_model_path'] = os.path.join( self.train_args['base_dir'], self.model_args['load'] + f'-{self.model_index}') # ENV self.env = make_env(self.env_args.to_dict, self.use_GCN) # ALGORITHM CONFIG Model, algorithm_config, _policy_mode = get_model_info( self.model_args['algo']) self.model_args['policy_mode'] = _policy_mode if self.model_args['algo_config'] is not None: algorithm_config = UpdateConfig(algorithm_config, self.model_args['algo_config'], 'algo') ShowConfig(algorithm_config) # BUFFER if _policy_mode == 'off-policy': self.buffer_args['batch_size'] = algorithm_config['batch_size'] self.buffer_args['buffer_size'] = algorithm_config['buffer_size'] if self.model_args['algo'] in ['drqn', 'drdqn']: self.buffer_args['type'] = 'EpisodeER' else: _use_priority = algorithm_config.get('use_priority', False) _n_step = algorithm_config.get('n_step', False) if _use_priority and _n_step: self.buffer_args['type'] = 'NstepPER' self.buffer_args['NstepPER'][ 'max_episode'] = self.train_args['max_episode'] self.buffer_args['NstepPER']['gamma'] = algorithm_config[ 'gamma'] algorithm_config['gamma'] = pow( algorithm_config['gamma'], self.buffer_args['NstepPER'] ['n']) # update gamma for n-step training. elif _use_priority: self.buffer_args['type'] = 'PER' self.buffer_args['PER']['max_episode'] = self.train_args[ 'max_episode'] elif _n_step: self.buffer_args['type'] = 'NstepER' self.buffer_args['NstepER']['gamma'] = algorithm_config[ 'gamma'] algorithm_config['gamma'] = pow( algorithm_config['gamma'], self.buffer_args['NstepER']['n']) else: self.buffer_args['type'] = 'ER' else: self.buffer_args['type'] = 'Pandas' # MODEL base_dir = os.path.join( self.train_args['base_dir'], self.train_args['name'] ) # train_args['base_dir'] DIR/ENV_NAME/ALGORITHM_NAME if 'batch_size' in algorithm_config.keys() and train_args['fill_in']: self.train_args['pre_fill_steps'] = algorithm_config['batch_size'] if self.env_args['type'] == 'gym': self.eval_env_args = deepcopy(self.env_args) self.eval_env_args.env_num = 1 self.eval_env = make_env(self.eval_env_args.to_dict) # buffer ------------------------------ if 'Nstep' in self.buffer_args[ 'type'] or 'Episode' in self.buffer_args['type']: self.buffer_args[self.buffer_args['type']][ 'agents_num'] = self.env_args['env_num'] self.buffer = get_buffer(self.buffer_args) # buffer ------------------------------ # model ------------------------------- model_params = { 's_dim': self.env.s_dim, 'visual_sources': self.env.visual_sources, 'visual_resolution': self.env.visual_resolution, 'a_dim_or_list': self.env.a_dim_or_list, 'is_continuous': self.env.is_continuous, 'max_episode': self.train_args.max_episode, 'base_dir': base_dir, 'logger2file': self.model_args.logger2file, 'seed': self.model_args.seed } self.model = Model(**model_params, **algorithm_config) self.model.set_buffer(self.buffer) self.model.init_or_restore(self.train_args['load_model_path']) # model ------------------------------- self.train_args['begin_episode'] = self.model.get_init_episode() if not self.train_args['inference']: records_dict = { 'env': self.env_args.to_dict, 'model': self.model_args.to_dict, 'buffer': self.buffer_args.to_dict, 'train': self.train_args.to_dict, 'algo': algorithm_config } save_config(os.path.join(base_dir, 'config'), records_dict) else: # buffer ----------------------------------- self.buffer_args_s = [] for i in range(self.env.brain_num): _bargs = deepcopy(self.buffer_args) if 'Nstep' in _bargs['type'] or 'Episode' in _bargs['type']: _bargs[_bargs['type']][ 'agents_num'] = self.env.brain_agents[i] self.buffer_args_s.append(_bargs) buffers = [ get_buffer(self.buffer_args_s[i]) for i in range(self.env.brain_num) ] # buffer ----------------------------------- # model ------------------------------------ self.model_args_s = [] for i in range(self.env.brain_num): _margs = deepcopy(self.model_args) _margs['seed'] = self.model_args['seed'] + i * 10 self.model_args_s.append(_margs) model_params = [ { 's_dim': self.env.s_dim[i], 'a_dim_or_list': self.env.a_dim_or_list[i], 'visual_sources': self.env.visual_sources[i], 'visual_resolution': self.env.visual_resolutions[i], 'is_continuous': self.env.is_continuous[i], 'max_episode': self.train_args.max_episode, 'base_dir': os.path.join(base_dir, b), 'logger2file': self.model_args_s[i].logger2file, 'seed': self.model_args_s[i]. seed, # 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 } for i, b in enumerate(self.env.brain_names) ] # multi agent training------------------------------------ if self.model_args['algo'][:3] == 'ma_': self.ma = True assert self.env.brain_num > 1, 'if using ma* algorithms, number of brains must larger than 1' self.ma_data = ExperienceReplay(batch_size=10, capacity=1000) [ mp.update({ 'n': self.env.brain_num, 'i': i }) for i, mp in enumerate(model_params) ] else: self.ma = False # multi agent training------------------------------------ self.models = [ Model(**model_params[i], **algorithm_config) for i in range(self.env.brain_num) ] [ model.set_buffer(buffer) for model, buffer in zip(self.models, buffers) ] [ self.models[i].init_or_restore( os.path.join(self.train_args['load_model_path'], b)) for i, b in enumerate(self.env.brain_names) ] # model ------------------------------------ self.train_args['begin_episode'] = self.models[0].get_init_episode( ) if not self.train_args['inference']: for i, b in enumerate(self.env.brain_names): records_dict = { 'env': self.env_args.to_dict, 'model': self.model_args_s[i].to_dict, 'buffer': self.buffer_args_s[i].to_dict, 'train': self.train_args.to_dict, 'algo': algorithm_config } save_config(os.path.join(base_dir, b, 'config'), records_dict)
def gym_run(default_args, share_args, options, max_step, max_episode, save_frequency, name): from gym_loop import Loop from gym.spaces import Box, Discrete, Tuple from gym_wrapper import gym_envs try: tf_version, (model, policy_mode, _) = get_model_info(options['--algorithm']) algorithm_config = sth.load_config( f'./Algorithms/{tf_version}/config.yaml')[options['--algorithm']] except KeyError: raise NotImplementedError available_type = [Box, Discrete] render_episode = int(options['--render-episode']) if options[ '--render-episode'] != 'None' else default_args['render_episode'] try: env = gym_envs(options['--gym-env'], int(options['--gym-agents'])) assert type(env.observation_space) in available_type and type( env.action_space ) in available_type, 'action_space and observation_space must be one of available_type' except Exception as e: print(e) if options['--config-file'] != 'None': algorithm_config = update_config(algorithm_config, options['--config-file']) _base_dir = os.path.join(share_args['base_dir'], options['--gym-env'], options['--algorithm']) base_dir = os.path.join(_base_dir, name) show_config(algorithm_config) if type(env.observation_space) == Box: s_dim = env.observation_space.shape[0] if len( env.observation_space.shape) == 1 else 0 else: s_dim = int(env.observation_space.n) if len(env.observation_space.shape) == 3: visual_sources = 1 visual_resolution = list(env.observation_space.shape) else: visual_sources = 0 visual_resolution = [] if type(env.action_space) == Box: assert len( env.action_space.shape ) == 1, 'if action space is continuous, the shape length of action must equal to 1' a_dim_or_list = env.action_space.shape action_type = 'continuous' elif type(env.action_space) == Tuple: assert all( [type(i) == Discrete for i in env.action_space] ) == True, 'if action space is Tuple, each item in it must have type Discrete' a_dim_or_list = [i.n for i in env.action_space] action_type = 'discrete' else: a_dim_or_list = [env.action_space.n] action_type = 'discrete' gym_model = model(s_dim=s_dim, visual_sources=visual_sources, visual_resolution=visual_resolution, a_dim_or_list=a_dim_or_list, action_type=action_type, max_episode=max_episode, base_dir=base_dir, logger2file=share_args['logger2file'], out_graph=share_args['out_graph'], **algorithm_config) gym_model.init_or_restore( os.path.join( _base_dir, name if options['--load'] == 'None' else options['--load'])) begin_episode = gym_model.get_init_episode() params = { 'env': env, 'gym_model': gym_model, 'action_type': action_type, 'begin_episode': begin_episode, 'save_frequency': save_frequency, 'max_step': max_step, 'max_episode': max_episode, 'eval_while_train': default_args['eval_while_train'], # whether to eval while training. 'max_eval_episode': default_args['max_eval_episode'], 'render': default_args['render'], 'render_episode': render_episode, 'policy_mode': policy_mode } if 'batch_size' in algorithm_config.keys() and options['--fill-in']: steps = algorithm_config['batch_size'] else: steps = default_args['random_steps'] if options['--inference']: Loop.inference(env, gym_model, action_type) else: sth.save_config(os.path.join(base_dir, 'config'), algorithm_config) try: Loop.no_op(env, gym_model, action_type, steps, choose=options['--noop-choose']) Loop.train(**params) except Exception as e: print(e) finally: try: gym_model.close() except Exception as e: print(e) finally: env.close() sys.exit()
def unity_run(default_args, share_args, options, max_step, max_episode, save_frequency, name): from mlagents.envs import UnityEnvironment from utils.sampler import create_sampler_manager try: tf_version, (model, policy_mode, _) = get_model_info(options['--algorithm']) algorithm_config = sth.load_config( f'./Algorithms/{tf_version}/config.yaml')[options['--algorithm']] ma = options['--algorithm'][:3] == 'ma_' except KeyError: raise NotImplementedError reset_config = default_args['reset_config'] if options['--unity']: env = UnityEnvironment() env_name = 'unity' else: file_name = default_args['exe_file'] if options[ '--env'] == 'None' else options['--env'] if os.path.exists(file_name): env = UnityEnvironment(file_name=file_name, base_port=int(options['--port']), no_graphics=False if options['--inference'] else not options['--graphic']) env_dir = os.path.split(file_name)[0] env_name = os.path.join(*env_dir.replace('\\', '/').replace( r'//', r'/').split('/')[-2:]) sys.path.append(env_dir) if os.path.exists(env_dir + '/env_config.py'): import env_config reset_config = env_config.reset_config max_step = env_config.max_step if os.path.exists(env_dir + '/env_loop.py'): from env_loop import Loop else: raise Exception('can not find this file.') sampler_manager, resampling_interval = create_sampler_manager( options['--sampler'], env.reset_parameters) if 'Loop' not in locals().keys(): if ma: from ma_loop import Loop else: from loop import Loop if options['--config-file'] != 'None': algorithm_config = update_config(algorithm_config, options['--config-file']) _base_dir = os.path.join(share_args['base_dir'], env_name, options['--algorithm']) base_dir = os.path.join(_base_dir, name) show_config(algorithm_config) brain_names = env.external_brain_names brains = env.brains brain_num = len(brain_names) visual_resolutions = {} for i in brain_names: if brains[i].number_visual_observations: visual_resolutions[f'{i}'] = [ brains[i].camera_resolutions[0]['height'], brains[i].camera_resolutions[0]['width'], 1 if brains[i].camera_resolutions[0]['blackAndWhite'] else 3 ] else: visual_resolutions[f'{i}'] = [] model_params = [{ 's_dim': brains[i].vector_observation_space_size * brains[i].num_stacked_vector_observations, 'a_dim_or_list': brains[i].vector_action_space_size, 'action_type': brains[i].vector_action_space_type, 'max_episode': max_episode, 'base_dir': os.path.join(base_dir, i), 'logger2file': share_args['logger2file'], 'out_graph': share_args['out_graph'], } for i in brain_names] if ma: assert brain_num > 1, 'if using ma* algorithms, number of brains must larger than 1' data = ExperienceReplay(share_args['ma']['batch_size'], share_args['ma']['capacity']) extra_params = {'data': data} models = [ model(n=brain_num, i=i, **model_params[i], **algorithm_config) for i in range(brain_num) ] else: extra_params = {} models = [ model(visual_sources=brains[i].number_visual_observations, visual_resolution=visual_resolutions[f'{i}'], **model_params[index], **algorithm_config) for index, i in enumerate(brain_names) ] [ models[index].init_or_restore( os.path.join( _base_dir, name if options['--load'] == 'None' else options['--load'], i)) for index, i in enumerate(brain_names) ] begin_episode = models[0].get_init_episode() params = { 'env': env, 'brain_names': brain_names, 'models': models, 'begin_episode': begin_episode, 'save_frequency': save_frequency, 'reset_config': reset_config, 'max_step': max_step, 'max_episode': max_episode, 'sampler_manager': sampler_manager, 'resampling_interval': resampling_interval, 'policy_mode': policy_mode } if 'batch_size' in algorithm_config.keys() and options['--fill-in']: steps = algorithm_config['batch_size'] else: steps = default_args['no_op_steps'] no_op_params = { 'env': env, 'brain_names': brain_names, 'models': models, 'brains': brains, 'steps': steps, 'choose': options['--noop-choose'] } params.update(extra_params) no_op_params.update(extra_params) if options['--inference']: Loop.inference(env, brain_names, models, reset_config=reset_config, sampler_manager=sampler_manager, resampling_interval=resampling_interval) else: try: [ sth.save_config(os.path.join(base_dir, i, 'config'), algorithm_config) for i in brain_names ] Loop.no_op(**no_op_params) Loop.train(**params) except Exception as e: print(e) finally: try: [models[i].close() for i in range(len(models))] except Exception as e: print(e) finally: env.close() sys.exit()
def gym_run(default_args, share_args, options, max_step, max_episode, save_frequency, name, seed): from gym_loop import Loop from gym_wrapper import gym_envs model, algorithm_config, policy_mode = get_model_info(options['--algorithm']) render_episode = int(options['--render-episode']) if options['--render-episode'] != 'None' else default_args['render_episode'] try: env = gym_envs(gym_env_name=options['--gym-env'], n=int(options['--gym-agents']), seed=int(options['--gym-env-seed']), render_mode=default_args['render_mode']) except Exception as e: print(e) if options['--config-file'] != 'None': algorithm_config = update_config(algorithm_config, options['--config-file']) _base_dir = os.path.join(share_args['base_dir'], options['--gym-env'], options['--algorithm']) base_dir = os.path.join(_base_dir, name) show_config(algorithm_config) model_params = { 's_dim': env.s_dim, 'visual_sources': env.visual_sources, 'visual_resolution': env.visual_resolution, 'a_dim_or_list': env.a_dim_or_list, 'is_continuous': env.is_continuous, 'max_episode': max_episode, 'base_dir': base_dir, 'logger2file': share_args['logger2file'], 'seed': seed, } gym_model = model( **model_params, **algorithm_config ) gym_model.init_or_restore(os.path.join(_base_dir, name if options['--load'] == 'None' else options['--load'])) begin_episode = gym_model.get_init_episode() params = { 'env': env, 'gym_model': gym_model, 'begin_episode': begin_episode, 'save_frequency': save_frequency, 'max_step': max_step, 'max_episode': max_episode, 'eval_while_train': default_args['eval_while_train'], # whether to eval while training. 'max_eval_episode': default_args['max_eval_episode'], 'render': default_args['render'], 'render_episode': render_episode, 'policy_mode': policy_mode } if 'batch_size' in algorithm_config.keys() and options['--fill-in']: steps = algorithm_config['batch_size'] else: steps = default_args['random_steps'] if options['--inference']: Loop.inference(env, gym_model) else: sth.save_config(os.path.join(base_dir, 'config'), algorithm_config) try: Loop.no_op(env, gym_model, steps, choose=options['--noop-choose']) Loop.train(**params) except Exception as e: print(e) finally: gym_model.close() env.close() sys.exit()