def set_policy(self):
self.policy = TD3(
actor_input_dim=self.actor_input_dim,
actor_output_dim=self.actor_output_dim,
critic_input_dim=self.critic_input_dim,
n_hidden=400,
max_action=self.max_action,
name=self.name,
args=self.args,
i_agent=self.i_agent)
from vrepsim import VrepSim
import time
from evaluate import evaluate_policy
from td3.td3 import TD3
if __name__ == '__main__':
agent = TD3()
sim = VrepSim()
sim.reset_sim()
agent.load()
evaluate_policy(agent, sim, eval_episodes=10, episode_length=50)
sim.reset_sim()
def main():
if sys.platform.startswith('win'):
# Add the _win_handler function to the windows console's handler function list
win32api.SetConsoleCtrlHandler(_win_handler, True)
if os.path.exists(
os.path.join(config_file.config['config_file'], 'config.yaml')):
config = sth.load_config(config_file.config['config_file'])
else:
config = config_file.config
print(f'load config from config.')
hyper_config = config['hyper parameters']
train_config = config['train config']
record_config = config['record config']
basic_dir = record_config['basic_dir']
last_name = record_config['project_name'] + '/' \
+ record_config['remark'] \
+ record_config['run_id']
cp_dir = record_config['checkpoint_basic_dir'] + last_name
cp_file = cp_dir + '/rb'
log_dir = record_config['log_basic_dir'] + last_name
excel_dir = record_config['excel_basic_dir'] + last_name
config_dir = record_config['config_basic_dir'] + last_name
sth.check_or_create(basic_dir, 'basic')
sth.check_or_create(cp_dir, 'checkpoints')
sth.check_or_create(log_dir, 'logs(summaries)')
sth.check_or_create(excel_dir, 'excel')
sth.check_or_create(config_dir, 'config')
logger = create_logger(
name='logger',
console_level=logging.INFO,
console_format='%(levelname)s : %(message)s',
logger2file=record_config['logger2file'],
file_name=log_dir + '\log.txt',
file_level=logging.WARNING,
file_format=
'%(lineno)d - %(asctime)s - %(module)s - %(funcName)s - %(levelname)s - %(message)s'
)
if train_config['train']:
sth.save_config(config_dir, config)
if train_config['unity_mode']:
env = UnityEnvironment()
else:
env = UnityEnvironment(
file_name=train_config['unity_file'],
no_graphics=True if train_config['train'] else False,
base_port=train_config['port'])
brain_name = env.external_brain_names[0]
brain = env.brains[brain_name]
# set the memory use proportion of GPU
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
# tf_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.reset_default_graph()
graph = tf.Graph()
with graph.as_default() as g:
with tf.Session(graph=g, config=tf_config) as sess:
logger.info('Algorithm: {0}'.format(
train_config['algorithm'].name))
if train_config['algorithm'] == config_file.algorithms.ppo_sep_ac:
from ppo.ppo_base import PPO_SEP
model = PPO_SEP(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('PPO_SEP initialize success.')
elif train_config['algorithm'] == config_file.algorithms.ppo_com:
from ppo.ppo_base import PPO_COM
model = PPO_COM(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('PPO_COM initialize success.')
elif train_config['algorithm'] == config_file.algorithms.sac:
from sac.sac import SAC
model = SAC(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('SAC initialize success.')
elif train_config['algorithm'] == config_file.algorithms.sac_no_v:
from sac.sac_no_v import SAC_NO_V
model = SAC_NO_V(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('SAC_NO_V initialize success.')
elif train_config['algorithm'] == config_file.algorithms.ddpg:
from ddpg.ddpg import DDPG
model = DDPG(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('DDPG initialize success.')
elif train_config['algorithm'] == config_file.algorithms.td3:
from td3.td3 import TD3
model = TD3(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
logger.info('TD3 initialize success.')
recorder = Recorder(log_dir,
excel_dir,
record_config,
logger,
max_to_keep=5,
pad_step_number=True,
graph=g)
episode = init_or_restore(cp_dir, sess, recorder, cp_file)
try:
if train_config['train']:
train_OnPolicy(
sess=sess,
env=env,
brain_name=brain_name,
begin_episode=episode,
model=model,
recorder=recorder,
cp_file=cp_file,
hyper_config=hyper_config,
train_config=train_config) if not train_config[
'use_replay_buffer'] else train_OffPolicy(
sess=sess,
env=env,
brain_name=brain_name,
begin_episode=episode,
model=model,
recorder=recorder,
cp_file=cp_file,
hyper_config=hyper_config,
train_config=train_config)
tf.train.write_graph(g,
cp_dir,
'raw_graph_def.pb',
as_text=False)
export_model(cp_dir, g)
else:
inference(env, brain_name, model, train_config)
except Exception as e:
logger.error(e)
finally:
env.close()
recorder.close()
sys.exit()
from td3.experience.priority_replay_buffer import PrioritizedReplayBuffer
from td3.experience.replay_buffer import ReplayBuffer
from td3.td3 import TD3
from td3.train import train
from td3.populate import populate_buffer, populate_buffer_zeros
from td3.experience.schedules import LinearSchedule
from td3 import set_mode
import argparse
if __name__ == '__main__':
# Set seeds
if cons.set_seed:
torch.manual_seed(cons.SEED)
np.random.seed(cons.SEED)
agent = TD3(set_mode.MODE)
sim = VrepSim()
sim.reset_sim()
if cons.PRIORITY:
replay_buffer = PrioritizedReplayBuffer(cons.BUFFER_SIZE,
alpha=cons.ALPHA)
if cons.BETA_ITERS is None:
cons.BETA_ITERS = cons.EXPLORATION
cons.BETA_SCHED = LinearSchedule(cons.BETA_ITERS,
initial_p=cons.BETA,
final_p=1.0)
else:
replay_buffer = ReplayBuffer()
def main():
if sys.platform.startswith('win'):
win32api.SetConsoleCtrlHandler(_win_handler, True)
if train_config['unity_mode']:
env = UnityEnvironment()
else:
env = UnityEnvironment(
file_name=train_config['unity_file'],
no_graphics=True if train_config['train'] else False,
base_port=train_config['port'])
brain_name = env.external_brain_names[0]
brain = env.brains[brain_name]
# set the memory use proportion of GPU
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
# tf_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.reset_default_graph()
graph = tf.Graph()
with graph.as_default() as g:
with tf.Session(graph=g, config=tf_config) as sess:
print('Algorithm: {0}'.format(train_config['algorithm'].name))
if train_config['algorithm'] == algorithms.ppo_sep_ac:
from ppo.ppo_base import PPO_SEP
model = PPO_SEP(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('PPO_SEP initialize success.')
elif train_config['algorithm'] == algorithms.ppo_com:
from ppo.ppo_base import PPO_COM
model = PPO_COM(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('PPO_COM initialize success.')
elif train_config['algorithm'] == algorithms.sac:
from sac.sac import SAC
model = SAC(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('SAC initialize success.')
elif train_config['algorithm'] == algorithms.sac_no_v:
from sac.sac_no_v import SAC_NO_V
model = SAC_NO_V(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('SAC_NO_V initialize success.')
elif train_config['algorithm'] == algorithms.ddpg:
from ddpg.ddpg import DDPG
model = DDPG(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('DDPG initialize success.')
elif train_config['algorithm'] == algorithms.td3:
from td3.td3 import TD3
model = TD3(sess=sess,
s_dim=brain.vector_observation_space_size,
a_counts=brain.vector_action_space_size[0],
hyper_config=hyper_config)
print('TD3 initialize success.')
sess.run(tf.global_variables_initializer())
try:
if train_config['train']:
train_OnPolicy(
sess=sess,
env=env,
brain_name=brain_name,
begin_episode=0,
model=model,
hyper_config=hyper_config,
train_config=train_config) if not train_config[
'use_replay_buffer'] else train_OffPolicy(
sess=sess,
env=env,
brain_name=brain_name,
begin_episode=0,
model=model,
hyper_config=hyper_config,
train_config=train_config)
else:
inference(env, brain_name, model, train_config)
except Exception as e:
print(e)
finally:
env.close()
sys.exit()