def main(args):
args.batch_size = None
logger.debug('CONFIGURATION: {}'.format(args))
""" Set up the graph, the agents, and run the agents in parallel. """
if args.env == 'GYM':
from environments import atari_environment
num_actions = atari_environment.get_num_actions(args.game)
input_shape = atari_environment.get_input_shape(args.game)
else:
num_actions = get_num_actions(args.rom_path, args.game)
args.summ_base_dir = '/tmp/summary_logs/{}/{}'.format(
args.game, time.strftime('%m.%d/%H.%M'))
logger.info('logging summaries to {}'.format(args.summ_base_dir))
Learner, Network = ALGORITHMS[args.alg_type]
network = Network({
'name': 'shared_vars_network',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
#initialize shared variables
args.learning_vars = SharedVars(network)
args.opt_state = SharedVars(
network, opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
args.batch_opt_state = SharedVars(
network, opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
if args.alg_type in ['q', 'sarsa', 'dueling', 'dqn-cts']:
args.target_vars = SharedVars(network)
args.target_update_flags = SharedFlags(args.num_actor_learners)
args.barrier = Barrier(args.num_actor_learners)
args.episode_counter = SharedCounter(0)
args.global_step = SharedCounter(0)
args.num_actions = num_actions
#spin up processes and block
if (args.visualize == 2): args.visualize = 0
actor_learners = []
experience_queue = Queue()
for i in xrange(args.num_actor_learners):
if (args.visualize == 2) and (i == args.num_actor_learners - 1):
args.args.visualize = 1
args.actor_id = i
rng = np.random.RandomState(int(time.time()))
args.random_seed = rng.randint(1000)
#pass in gpu name to learner here and wrap each learner in device context
args.queue = experience_queue #only used by TRPO
args.input_shape = input_shape
actor_learners.append(Learner(args))
actor_learners[-1].start()
for t in actor_learners:
t.join()
logger.info('All training threads finished!')
args = parser.parse_args()
setup_loggings(args)
torch.manual_seed(args.seed)
env = create_atari_env(args.env_name)
shared_model = ActorCritic(env.observation_space.shape[0],
env.action_space)
shared_model.share_memory()
if args.no_shared:
optimizer = None
else:
optimizer = my_optim.SharedAdam(shared_model.parameters(), lr=args.lr)
optimizer.share_memory()
gl_step_cnt = SharedCounter()
if not args.debug:
processes = []
p = mp.Process(target=test,
args=(args.num_processes, args, shared_model,
gl_step_cnt))
p.start()
processes.append(p)
for rank in range(0, args.num_processes):
p = mp.Process(target=train,
args=(rank, args, shared_model, gl_step_cnt,
optimizer))
p.start()
processes.append(p)
def main(args):
args.batch_size = None
logger.debug('CONFIGURATION: {}'.format(args))
""" Set up the graph, the agents, and run the agents in parallel. """
if args.env == 'GYM':
from environments import atari_environment
num_actions = atari_environment.get_num_actions(args.game)
input_shape = atari_environment.get_input_shape(args.game)
else:
num_actions = get_num_actions(args.rom_path, args.game)
args.summ_base_dir = '/tmp/summary_logs/{}/{}'.format(
args.game, time.strftime('%m.%d/%H.%M'))
logger.info('logging summaries to {}'.format(args.summ_base_dir))
algorithms = {
'q': (NStepQLearner, QNetwork),
'sarsa': (OneStepSARSALearner, QNetwork),
'dueling': (DuelingLearner, DuelingNetwork),
'a3c': (A3CLearner, PolicyValueNetwork),
'a3c-lstm': (A3CLSTMLearner, PolicyValueNetwork),
'a3c-sequence-decoder':
(ActionSequenceA3CLearner, SequencePolicyVNetwork),
'pgq': (PGQLearner, PolicyValueNetwork),
'pgq-lstm': (PGQLSTMLearner, PolicyValueNetwork),
'trpo': (TRPOLearner, PolicyNetwork),
'cem': (CEMLearner, PolicyNetwork),
'q-cts': (PseudoCountQLearner, QNetwork),
'a3c-cts': (PseudoCountA3CLearner, PolicyValueNetwork),
'a3c-repeat': (ARA3CLearner, PolicyRepeatNetwork),
}
assert args.alg_type in algorithms, 'alg_type `{}` not implemented'.format(
args.alg_type)
Learner, Network = algorithms[args.alg_type]
network = Network({
'name': 'shared_vars_network',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
args.learning_vars = SharedVars(num_actions, args.alg_type, network)
args.opt_state = SharedVars(
num_actions,
args.alg_type,
network,
opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
args.batch_opt_state = SharedVars(
num_actions,
args.alg_type,
network,
opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
if args.alg_type in ['q', 'sarsa', 'dueling', 'q-cts']:
args.target_vars = SharedVars(num_actions, args.alg_type, network)
args.target_update_flags = SharedFlags(args.num_actor_learners)
args.barrier = Barrier(args.num_actor_learners)
args.episode_counter = SharedCounter(0)
args.global_step = SharedCounter(0)
args.num_actions = num_actions
if (args.visualize == 2): args.visualize = 0
actor_learners = []
experience_queue = Queue()
for i in xrange(args.num_actor_learners):
if (args.visualize == 2) and (i == args.num_actor_learners - 1):
args.args.visualize = 1
args.actor_id = i
rng = np.random.RandomState(int(time.time()))
args.random_seed = rng.randint(1000)
#pass in gpu name to learner here and wrap each learner in device context
args.queue = experience_queue #only used by TRPO
args.input_shape = input_shape
actor_learners.append(Learner(args))
actor_learners[-1].start()
for t in actor_learners:
t.join()
logger.info('All training threads finished!')
def main(args):
args.batch_size = None
logger.debug('CONFIGURATION: {}'.format(args))
""" Set up the graph, the agents, and run the agents in parallel. """
if args.env == 'GYM':
from environments import atari_environment
num_actions, action_space, _ = atari_environment.get_actions(args.game)
input_shape = atari_environment.get_input_shape(args.game)
elif args.env == 'DOOM':
from environments.vizdoom_env import VizDoomEnv
env = VizDoomEnv(args.doom_cfg, args.game, args.is_train)
num_actions, action_space = env.get_actions()
input_shape = env.get_input_shape()
else:
num_actions = get_num_actions(args.rom_path, args.game)
args.action_space = action_space
args.summ_base_dir = '/tmp/summary_logs/{}/{}'.format(
args.game, time.strftime('%m.%d/%H.%M'))
logger.info('logging summaries to {}'.format(args.summ_base_dir))
Learner, Network = ALGORITHMS[args.alg_type]
network = Network({
'name': 'shared_vars_network',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
args.network = Network
#initialize shared variables
args.learning_vars = SharedVars(network.params)
args.opt_state = SharedVars(
network.params, opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
args.batch_opt_state = SharedVars(
network.params, opt_type=args.opt_type,
lr=args.initial_lr) if args.opt_mode == 'shared' else None
#TODO: need to refactor so TRPO+GAE doesn't need special treatment
if args.alg_type in ['trpo', 'trpo-continuous']:
if args.arch == 'FC': #add timestep feature
vf_input_shape = [input_shape[0] + 1]
else:
vf_input_shape = input_shape
baseline_network = PolicyValueNetwork(
{
'name': 'shared_value_network',
'input_shape': vf_input_shape,
'num_act': num_actions,
'args': args
},
use_policy_head=False)
args.baseline_vars = SharedVars(baseline_network.params)
args.vf_input_shape = vf_input_shape
if args.alg_type in ['q', 'sarsa', 'dueling', 'dqn-cts']:
args.target_vars = SharedVars(network.params)
args.target_update_flags = SharedFlags(args.num_actor_learners)
if args.alg_type == 'dqn-cts':
args.density_model_update_flags = SharedFlags(args.num_actor_learners)
tf.reset_default_graph()
args.barrier = Barrier(args.num_actor_learners)
args.global_step = SharedCounter(0)
args.num_actions = num_actions
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
num_gpus = 0
if cuda_visible_devices:
num_gpus = len(cuda_visible_devices.split())
#spin up processes and block
if (args.visualize == 2): args.visualize = 0
actor_learners = []
task_queue = Queue()
experience_queue = Queue()
seed = args.seed or np.random.randint(2**32)
np.random.seed(seed)
tf.set_random_seed(seed)
for i in xrange(args.num_actor_learners):
if (args.visualize == 2) and (i == args.num_actor_learners - 1):
args.args.visualize = 1
args.actor_id = i
args.device = '/gpu:{}'.format(i % num_gpus) if num_gpus else '/cpu:0'
args.random_seed = seed + i
#only used by TRPO
args.task_queue = task_queue
args.experience_queue = experience_queue
args.input_shape = input_shape
actor_learners.append(Learner(args))
actor_learners[-1].start()
try:
for t in actor_learners:
t.join()
except KeyboardInterrupt:
#Terminate with extreme prejudice
for t in actor_learners:
t.terminate()
logger.info('All training threads finished!')
logger.info('Use seed={} to reproduce'.format(seed))
def main(args):
args.batch_size = None
logger.debug('CONFIGURATION: {}'.format(args))
""" Set up the graph, the agents, and run the agents in parallel. """
if args.env == 'GYM':
from environments import atari_environment
num_actions, action_space, _ = atari_environment.get_actions(args.game)
input_shape = atari_environment.get_input_shape(args.game)
else:
num_actions = get_num_actions(args.rom_path, args.game)
args.action_space = action_space
args.summ_base_dir = '/tmp/summary_logs/{}/{}'.format(args.game, time.strftime('%m.%d/%H.%M'))
logger.info('logging summaries to {}'.format(args.summ_base_dir))
Learner, Network = ALGORITHMS[args.alg_type]
#print("Learner is: {}".format(Learner))
if args.alg_type !='AE':
network = Network({
'name': 'shared_vars_network',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
args.network = Network
else:
network_lower = Network({
'name': 'shared_vars_network_lower',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
args.network_lower = Network
network_upper = Network({
'name': 'shared_vars_network_upper',
'input_shape': input_shape,
'num_act': num_actions,
'args': args
})
args.network_upper = Network
## initialize visdom server
args.visdom = visdom.Visdom(port=args.display_port, env='AE DQN')
#initialize shared variables
#TODO: !!!!!! only network lower params are being use, should check out if upper is also needed !!!!!!!
if args.alg_type !='AE':
args.learning_vars = SharedVars(network.params) #size, step and optimizer
args.opt_state = SharedVars(
network.params, opt_type=args.opt_type, lr=args.initial_lr
) if args.opt_mode == 'shared' else None
args.batch_opt_state = SharedVars(
network.params, opt_type=args.opt_type, lr=args.initial_lr
) if args.opt_mode == 'shared' else None
else:
#args.learning_vars = SharedVars(network_lower.params) #size, step and optimizer
args.learning_vars_lower = SharedVars(network_lower.params) #size, step and optimizer
args.learning_vars_upper = SharedVars(network_upper.params) #size, step and optimizer
args.opt_state_lower = SharedVars(
network_lower.params, opt_type=args.opt_type, lr=args.initial_lr
)
args.opt_state_upper = SharedVars(
network_upper.params, opt_type=args.opt_type, lr=args.initial_lr
) if args.opt_mode == 'shared' else None
args.batch_opt_state_lower = SharedVars(
network_lower.params, opt_type=args.opt_type, lr=args.initial_lr
)
args.batch_opt_state_uppper = SharedVars(
network_upper.params, opt_type=args.opt_type, lr=args.initial_lr
) if args.opt_mode == 'shared' else None
#TODO: need to refactor so TRPO+GAE doesn't need special treatment
if args.alg_type in ['trpo', 'trpo-continuous']:
if args.arch == 'FC': #add timestep feature
vf_input_shape = [input_shape[0]+1]
else:
vf_input_shape = input_shape
baseline_network = PolicyValueNetwork({
'name': 'shared_value_network',
'input_shape': vf_input_shape,
'num_act': num_actions,
'args': args
}, use_policy_head=False)
args.baseline_vars = SharedVars(baseline_network.params)
args.vf_input_shape = vf_input_shape
if args.alg_type in ['q', 'sarsa', 'dueling', 'dqn-cts']:
args.target_vars = SharedVars(network.params)
args.target_update_flags = SharedFlags(args.num_actor_learners)
if args.alg_type in ['dqn-cts', 'a3c-cts', 'a3c-lstm-cts']: #TODO check density_model_update_flags
args.density_model_update_flags = SharedFlags(args.num_actor_learners)
if args.alg_type in ['AE']:
#print("we are in main args.alg_type in [AE]")
args.target_vars_lower = SharedVars(network_lower.params)
args.target_vars_upper = SharedVars(network_upper.params)
args.target_update_flags = SharedFlags(args.num_actor_learners)
args.density_model_update_flags = SharedFlags(args.num_actor_learners)
tf.reset_default_graph()
args.barrier = Barrier(args.num_actor_learners)
args.global_step = SharedCounter(0)
#ars.shared_visualizer = Visualizer(args.num_actor_learners) ## TODO to make it shared between the processes
args.num_actions = num_actions
cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
num_gpus = 0
if cuda_visible_devices:
num_gpus = len(cuda_visible_devices.split())
#spin up processes and block
# if (args.visualize == 2): args.visualize = 0
actor_learners = []
task_queue = Queue()
experience_queue = Queue()
seed = args.seed or np.random.randint(2**32)
np.random.seed(seed)
tf.set_random_seed(seed)
visualize = args.visualize
for i in range(args.num_actor_learners):
if (visualize == 2) and (i == args.num_actor_learners - 1):
args.visualize = 1
else:
args.visualize = 0
args.actor_id = i
args.device = '/gpu:{}'.format(i % num_gpus) if num_gpus else '/cpu:0'
args.random_seed = seed + i
#only used by TRPO
args.task_queue = task_queue
args.experience_queue = experience_queue
args.input_shape = input_shape
actor_learners.append(Learner(args))
actor_learners[-1].start()
if i == 1:
setup_kill_signal_handler(actor_learners[-1])
try:
for t in actor_learners:
file_name = "myfile_"+str(t)
with open("grpah", 'w') as file_name:
wr = csv.writer(file_name, quoting=csv.QUOTE_ALL)
wr.writerow(t.vis.plot_data['X'])
wr.writerow(t.vis.plot_data['Y'])
print ('[%s]' % ', '.join(map(str, t.vis.plot_data['X'])))
t.join()
except KeyboardInterrupt:
#Terminate with extreme prejudice
for t in actor_learners:
t.terminate()
logger.info('All training threads finished!')
logger.info('Use seed={} to reproduce'.format(seed))
args.input_shape = env.observation_space.shape
#import ipdb; ipdb.set_trace()
args.num_actions = env.action_space.n
#args.num_actions = 3
env.close()
setup_loggings(args)
shared_model = Model(args.input_shape, args.num_actions)
shared_model.share_memory()
if args.no_shared:
optimizer = None
else:
optimizer = my_optim.SharedAdam(shared_model.parameters(), lr=args.lr)
optimizer.share_memory()
shared_stepcount = SharedCounter()
if not args.debug:
processes = []
p = mp.Process(target=test, args=(args.num_processes, args,
shared_model, Model, make_env, shared_stepcount))
p.start()
processes.append(p)
for rank in range(0, args.num_processes):
p = mp.Process(target=train, args=(rank, args, shared_model,
Model, make_env, shared_stepcount, optimizer))
p.start()
processes.append(p)
for p in processes:
p.join()