def f(ppo_policy, config, conditions): # env to generate fake state env = gym.make(config.env) env = BBallWrapper(env, init_mode=3, fps=config.FPS, if_back_real=config.if_back_real, time_limit=config.max_length) env = MonitorWrapper( env, directory=os.path.join(config.logdir, 'gail_training/'), if_back_real=config.if_back_real, # init from dataset in order init_mode=3) # align the conditions with env # -1 : newest state conditions = conditions[None] env.data = conditions[:, :, -1] obs_state = env.reset() one_epi_fake = [] one_epi_fake_act = [] for len_idx in range(config.max_length): if config.if_back_real: act = ppo_policy.act(np.array(conditions[:, len_idx:len_idx + 1, :]), stochastic=True) else: act = ppo_policy.act(np.array(obs_state)[None, None], stochastic=True) transformed_act = [ # Discrete(3) must be int int(0), # Box(2,) np.array([0.0, 0.0], dtype=np.float32), # Box(5, 2) np.zeros(shape=[5, 2], dtype=np.float32), # Box(5, 2) np.reshape(act, [5, 2]) ] obs_state, _, _, _ = env.step(transformed_act) one_epi_fake.append(obs_state[-1]) one_epi_fake_act.append(act.reshape([5, 2])) return one_epi_fake, one_epi_fake_act
def train(config, env_processes, outdir): """ Training and evaluation entry point yielding scores. Resolves some configuration attributes, creates environments, graph, and training loop. By default, assigns all operations to the CPU. Args ---- config : Object providing configurations via attributes. env_processes : Whether to step environment in external processes. outdir : Directory path to save rendering result while traning. Yields ------ score : Evaluation scores. """ tf.reset_default_graph() # env to get config dummy_env = gym.make(config.env) def normalize_observ(observ): min_ = dummy_env.observation_space.low[0] max_ = dummy_env.observation_space.high[0] observ = 2.0 * (observ - min_) / (max_ - min_) - 1.0 return observ def normalize_action(act): min_ = dummy_env.action_space[3].low max_ = dummy_env.action_space[3].high act = 2.0 * (act - min_) / (max_ - min_) - 1.0 return act def denormalize_observ(observ): min_ = dummy_env.observation_space.low[0] max_ = dummy_env.observation_space.high[0] observ = (observ + 1.0) * (max_ - min_) / 2.0 + min_ return observ # env to testing vanilla_env = gym.make(config.env) vanilla_env = BBallWrapper(vanilla_env, init_mode=1, fps=config.FPS, if_back_real=False, time_limit=50) vanilla_env = MonitorWrapper(vanilla_env, directory=os.path.join(config.logdir, 'gail_testing_{}/'.format(config.train_len)), if_back_real=False, video_callable=lambda _: True, # init from dataset init_mode=1) # if not os.path.exists(os.path.join(config.logdir, 'gail_testing')): # os.makedirs(os.path.join(config.logdir, 'gail_testing')) vanilla_env.data = np.load('bball_strategies/data/GAILEnvData_51.npy') # env to generate fake state env = gym.make(config.env) env = BBallWrapper(env, init_mode=3, fps=config.FPS, if_back_real=config.if_back_real, time_limit=config.max_length) env = MonitorWrapper(env, directory=os.path.join(config.logdir, 'gail_training/'), if_back_real=config.if_back_real, # init from dataset in order init_mode=3) # Discriminator graph with tf.device('/gpu:0'): D = Discriminator(config, dummy_env) # PPO graph if config.update_every % config.num_agents: tf.logging.warn('Number of agents should divide episodes per update.') with tf.device('/cpu:0'): batch_env = utility.define_batch_env( lambda: _create_environment(config), config.num_agents, env_processes, outdir=outdir, is_gail=config.is_gail) graph = utility.define_simulation_graph( batch_env, config.algorithm, config) loop = _define_loop( graph, config.logdir, config.update_every * config.max_length, config.eval_episodes * config.max_length) total_steps = int( config.steps / config.update_every * (config.update_every + config.eval_episodes)) # Agent to genrate acttion ppo_policy = PPOPolicy(config, env) # Data all_data = h5py.File( 'bball_strategies/data/GAILTransitionData_{}.hdf5'.format(config.train_len), 'r') expert_data, valid_expert_data = np.split( all_data['OBS'].value, [all_data['OBS'].value.shape[0]*9//10]) expert_action, valid_expert_action = np.split( all_data['DEF_ACT'].value, [all_data['DEF_ACT'].value.shape[0]*9//10]) print('expert_data', expert_data.shape) print('valid_expert_data', valid_expert_data.shape) print('expert_action', expert_action.shape) print('valid_expert_action', valid_expert_action.shape) # TF Session # TODO _num_finished_episodes => Variable:0 saver = utility.define_saver( exclude=(r'.*_temporary.*', r'.*memory.*', r'Variable:0', r'.*Adam.*', r'.*beta.*')) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=config.log_device_placement) sess_config.gpu_options.allow_growth = True with tf.Session(config=sess_config) as sess: utility.initialize_variables( sess, saver, config.logdir, resume=FLAGS.resume) # NOTE reset variables in optimizer D.reset_optimizer(sess) # reset PPO optimizer opt_reset = tf.group( [v.initializer for v in graph.algo._optimizer.variables()]) sess.run(opt_reset) # visulization stuff if FLAGS.tally_only: tally_reward_line_chart(config, sess.run( D._global_steps), ppo_policy, D, denormalize_observ, normalize_observ, normalize_action) exit() # GAIL cumulate_steps = sess.run(graph.step) episode_idx = 0 valid_episode_idx = 0 while True: if episode_idx > (expert_data.shape[0]-config.episodes_per_batch*config.train_d_per_ppo) or episode_idx == 0: episode_idx = 0 perm_idx = np.random.permutation(expert_data.shape[0]) expert_data = expert_data[perm_idx] expert_action = expert_action[perm_idx] if valid_episode_idx > (valid_expert_data.shape[0]-config.episodes_per_batch) or valid_episode_idx == 0: valid_episode_idx = 0 valid_perm_idx = np.random.permutation( valid_expert_data.shape[0]) valid_expert_data = valid_expert_data[valid_perm_idx] valid_expert_action = valid_expert_action[valid_perm_idx] # testing if valid_episode_idx % (100 * config.episodes_per_batch) == 0: test_policy(config, vanilla_env, sess.run(D._global_steps), ppo_policy, D, denormalize_observ) if valid_episode_idx % (1000 * config.episodes_per_batch) == 0: tally_reward_line_chart(config, sess.run( D._global_steps), ppo_policy, D, denormalize_observ, normalize_observ, normalize_action) # train Discriminator train_Discriminator( episode_idx, config, expert_data, expert_action, env, ppo_policy, D, normalize_observ, normalize_action) if valid_episode_idx % (1000 * config.episodes_per_batch) == 0: tally_reward_line_chart(config, sess.run( D._global_steps), ppo_policy, D, denormalize_observ, normalize_observ, normalize_action) # valid Discriminator valid_Discriminator( valid_episode_idx, config, valid_expert_data, valid_expert_action, env, ppo_policy, D, normalize_observ, normalize_action) episode_idx += config.episodes_per_batch*config.train_d_per_ppo valid_episode_idx += config.episodes_per_batch # train PPO print('train PPO') cumulate_steps += total_steps for score in loop.run(sess, saver, cumulate_steps): yield score batch_env.close() vanilla_env.close() env.close()
def tally_reward_line_chart(config, steps, ppo_policy, D, denormalize_observ, normalize_observ, normalize_action): """ tally 100 episodes as line chart to show how well the discriminator judge on each state of real and fake episode """ if config.is_gail: episode_amount = 100 # real data all_data = h5py.File( 'bball_strategies/data/GAILTransitionData_51.hdf5', 'r') expert_data, _ = np.split( all_data['OBS'].value, [all_data['OBS'].value.shape[0]*9//10]) expert_action, _ = np.split( all_data['DEF_ACT'].value, [all_data['DEF_ACT'].value.shape[0]*9//10]) # env vanilla_env = gym.make(config.env) vanilla_env = BBallWrapper(vanilla_env, init_mode=1, fps=config.FPS, if_back_real=False, time_limit=50) vanilla_env.data = np.load('bball_strategies/data/GAILEnvData_51.npy') # real selected_idx = np.random.choice(expert_data.shape[0], episode_amount) # frame 0 is condition batch_real_states = expert_data[selected_idx, 1:] real_action = expert_action[selected_idx, :-1] batch_real_states = np.concatenate( batch_real_states, axis=0) real_action = np.concatenate(real_action[:, None], axis=0) batch_real_states = normalize_observ(batch_real_states) real_rewards = D.get_rewards_value( batch_real_states, normalize_action(real_action)).reshape([100, -1]) # fake numpy_collector = [] act_collector = [] for _ in range(episode_amount): vanilla_obs = vanilla_env.reset() for _ in range(vanilla_env.time_limit): vanilla_act = ppo_policy.act( np.array(vanilla_obs)[None, None], stochastic=False) act_collector.append(vanilla_act.reshape([1, 5, 2])) vanilla_trans_act = [ # Discrete(3) must be int int(0), # Box(2,) np.array([0.0, 0.0], dtype=np.float32), # Box(5, 2) np.zeros(shape=[5, 2], dtype=np.float32), # Box(5, 2) np.reshape(vanilla_act, [5, 2]) ] vanilla_obs, _, _, _ = vanilla_env.step( vanilla_trans_act) numpy_collector.append(vanilla_obs) numpy_collector = np.array(numpy_collector) act_collector = np.array(act_collector) fake_rewards = D.get_rewards_value( numpy_collector, act_collector).reshape([100, -1]) # vis vis_line_chart(real_rewards, fake_rewards, config.logdir, str(steps)) else: episode_amount = 100 # real data all_data = h5py.File( 'bball_strategies/data/GAILTransitionData_51.hdf5', 'r') expert_data, _ = np.split( all_data['OBS'].value, [all_data['OBS'].value.shape[0]*9//10]) expert_action, _ = np.split( all_data['DEF_ACT'].value, [all_data['DEF_ACT'].value.shape[0]*9//10]) # env vanilla_env = gym.make(config.env) vanilla_env = BBallWrapper(vanilla_env, init_mode=1, fps=config.FPS, if_back_real=False, time_limit=config.max_length) vanilla_env.data = np.load('bball_strategies/data/GAILEnvData_51.npy') # real selected_idx = np.random.choice(expert_data.shape[0], episode_amount) # frame 0 is condition batch_real_states = expert_data[selected_idx, 1:config.max_length+1, -1] real_action = expert_action[selected_idx, :config.max_length] batch_real_states = normalize_observ(batch_real_states) real_rewards = D.get_rewards_value( batch_real_states, normalize_action(real_action)).reshape([-1, 1]) real_rewards = np.tile(real_rewards, [1, config.max_length]) # fake numpy_collector = [] act_collector = [] for _ in range(episode_amount): vanilla_obs = vanilla_env.reset() epi_obs = [] epi_act = [] for _ in range(config.max_length): vanilla_act = ppo_policy.act( np.array(vanilla_obs)[None, None], stochastic=False) vanilla_trans_act = [ # Discrete(3) must be int int(0), # Box(2,) np.array([0.0, 0.0], dtype=np.float32), # Box(5, 2) np.zeros(shape=[5, 2], dtype=np.float32), # Box(5, 2) np.reshape(vanilla_act, [5, 2]) ] vanilla_obs, _, _, _ = vanilla_env.step( vanilla_trans_act) epi_obs.append(vanilla_obs[-1]) epi_act.append(vanilla_act.reshape([5, 2])) numpy_collector.append(epi_obs) act_collector.append(epi_act) numpy_collector = np.array(numpy_collector) act_collector = np.array(act_collector) fake_rewards = D.get_rewards_value( numpy_collector, act_collector).reshape([-1, 1]) fake_rewards = np.tile(fake_rewards, [1, config.max_length]) # vis vis_line_chart(real_rewards, fake_rewards, config.logdir, str(steps))
def train(config, env_processes, outdir): """ Training and evaluation entry point yielding scores. Resolves some configuration attributes, creates environments, graph, and training loop. By default, assigns all operations to the CPU. Args ---- config : Object providing configurations via attributes. env_processes : Whether to step environment in external processes. outdir : Directory path to save rendering result while traning. Yields ------ score : Evaluation scores. """ tf.reset_default_graph() # env to get config dummy_env = gym.make(config.env) def normalize_observ(observ): min_ = dummy_env.observation_space.low[0] max_ = dummy_env.observation_space.high[0] observ = 2.0 * (observ - min_) / (max_ - min_) - 1.0 return observ def normalize_action(act): min_ = dummy_env.action_space[3].low max_ = dummy_env.action_space[3].high act = 2.0 * (act - min_) / (max_ - min_) - 1.0 return act def denormalize_observ(observ): min_ = dummy_env.observation_space.low[0] max_ = dummy_env.observation_space.high[0] observ = (observ + 1.0) * (max_ - min_) / 2.0 + min_ return observ # env to testing vanilla_env = gym.make(config.env) vanilla_env = BBallWrapper(vanilla_env, init_mode=1, fps=config.FPS, if_back_real=False, time_limit=50) vanilla_env = MonitorWrapper(vanilla_env, directory=os.path.join(config.logdir, 'gail_testing_G{}_D{}/'.format(config.train_len, config.D_len)), if_back_real=False, video_callable=lambda _: True, # init from dataset init_mode=1) vanilla_env.data = np.load('bball_strategies/data/GAILEnvData_51.npy') # env to generate fake state env = gym.make(config.env) env = BBallWrapper(env, init_mode=3, fps=config.FPS, if_back_real=config.if_back_real, time_limit=config.max_length) env = MonitorWrapper(env, directory=os.path.join(config.logdir, 'gail_training/'), if_back_real=config.if_back_real, # init from dataset in order init_mode=3) # PPO graph if config.update_every % config.num_agents: tf.logging.warn('Number of agents should divide episodes per update.') with tf.device('/cpu:0'): batch_env = utility.define_batch_env( lambda: _create_environment(config), config.num_agents, env_processes, outdir=outdir, is_gail=config.is_gail) graph = utility.define_simulation_graph( batch_env, config.algorithm, config) loop = _define_loop( graph, config.logdir, config.update_every * config.max_length, config.eval_episodes * config.max_length) total_steps = int( config.steps / config.update_every * (config.update_every + config.eval_episodes)) # Agent to genrate acttion ppo_policy = PPOPolicy(config, env) # Data all_data = h5py.File( 'bball_strategies/data/GAILTransitionData_{}.hdf5'.format(config.train_len), 'r') expert_data, valid_expert_data = np.split( all_data['OBS'].value, [all_data['OBS'].value.shape[0] * 9 // 10]) expert_action, valid_expert_action = np.split( all_data['DEF_ACT'].value, [all_data['DEF_ACT'].value.shape[0] * 9 // 10]) print('expert_data', expert_data.shape) print('valid_expert_data', valid_expert_data.shape) print('expert_action', expert_action.shape) print('valid_expert_action', valid_expert_action.shape) # Preprocessing/ Normalization expert_data = normalize_observ(expert_data) valid_expert_data = normalize_observ(valid_expert_data) expert_action = normalize_action(expert_action) valid_expert_action = normalize_action(valid_expert_action) # summary writer of Discriminator summary_writer = tf.summary.FileWriter(config.logdir + '/Disciminator') # TF Session # TODO _num_finished_episodes => Variable:0 saver = utility.define_saver( exclude=(r'.*_temporary.*', r'.*memory.*', r'Variable:0', r'.*Adam.*', r'.*beta.*')) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=config.log_device_placement) sess_config.gpu_options.allow_growth = True with tf.Session(config=sess_config) as sess: utility.initialize_variables( sess, saver, config.logdir, resume=FLAGS.resume) # NOTE reset variables in optimizer # opt_reset_D = tf.group( # [v.initializer for v in graph.algo.D.optimizer.variables()]) # # reset PPO optimizer # opt_reset = tf.group( # [v.initializer for v in graph.algo._optimizer.variables()]) # sess.run([opt_reset, opt_reset_D]) # visulization stuff if FLAGS.tally_only: tally_reward_line_chart(config, sess.run( graph.algo.D._steps), ppo_policy, D, denormalize_observ, normalize_observ, normalize_action) exit() # GAIL cumulate_steps = sess.run(graph.step) episode_idx = 0 while True: if episode_idx > (expert_data.shape[0] - config.episodes_per_batch * config.train_d_per_ppo) or episode_idx == 0: episode_idx = 0 perm_idx = np.random.permutation(expert_data.shape[0]) expert_data = expert_data[perm_idx] expert_action = expert_action[perm_idx] # # testing if episode_idx % (config.train_d_per_ppo * 100 * config.episodes_per_batch) == 0: test_policy(config, vanilla_env, sess.run(graph.algo.D._steps), ppo_policy, graph.algo.D, denormalize_observ) if episode_idx % (config.train_d_per_ppo * 1000 * config.episodes_per_batch) == 0: tally_reward_line_chart(config, sess.run( graph.algo.D._steps), ppo_policy, graph.algo.D, denormalize_observ, normalize_observ, normalize_action) # # train Discriminator gail_timer = time.time() for _ in range(config.train_d_per_ppo): if config.is_double_curiculum: observ = expert_data[episode_idx:episode_idx +config.episodes_per_batch, 1:] action = expert_action[episode_idx:episode_idx+config.episodes_per_batch, :-1] if config.use_padding: # 1. padding with buffer buffer = observ[:, 0, :-1] padded_observ = np.concatenate([buffer, observ[:, :, -1]], axis=1) padded_act = np.concatenate([np.zeros(shape=[action.shape[0], 9, 5, 2]), action], axis=1) # 2. split the whole episode into training data of Discriminator with length=config.D_len training_obs = [] training_act = [] for i in range(config.max_length-config.D_len+10): training_obs.append(padded_observ[:, i:i+config.D_len]) training_act.append(padded_act[:, i:i+config.D_len]) training_obs = np.concatenate(training_obs, axis=0) training_act = np.concatenate(training_act, axis=0) else: pass else: training_obs = expert_data[episode_idx:episode_idx +config.episodes_per_batch, 1:, -1] training_act = expert_action[episode_idx:episode_idx+config.episodes_per_batch, :-1] feed_dict = { graph.is_training: True, graph.should_log: True, graph.do_report: True, graph.force_reset: False, graph.algo.D._expert_s: training_obs, graph.algo.D._expert_a: training_act} gail_counter = 0 while gail_counter < config.gail_steps: gail_summary = sess.run( graph.gail_summary, feed_dict=feed_dict) if gail_summary: summary_writer.add_summary( gail_summary, global_step=sess.run(graph.algo.D._steps)) gail_counter += 1 episode_idx += config.episodes_per_batch print('Time Cost of Discriminator per Update: {}'.format( (time.time() - gail_timer) / config.train_d_per_ppo)) # train ppo cumulate_steps += total_steps for score in loop.run(sess, saver, cumulate_steps): yield score batch_env.close() vanilla_env.close() env.close()