def get_pg_train(name, pols, env, logdir, gamma, shape, lr, batch_size):
config = {
"gamma": gamma,
"sample_batch_size": batch_size,
"lr":
lr, #0.01 is too high, 0.0001 is too low , 0.001 seems to work (first 5 are 0.005, last are 0.001)
"multiagent": {
"policies": pols,
"policy_mapping_fn": policy_mapping_fn,
"policies_to_train": [name],
},
"model": {
"fcnet_activation": "tanh",
# Number of hidden layers for fully connected net
"fcnet_hiddens": shape,
},
# disable filters, otherwise we would need to synchronize those
# as well to the DQN agent
"observation_filter": "NoFilter",
"callbacks": {
"on_episode_end": partial(on_episode_end, gamma=gamma)
}
}
return PGTrainer(env=env,
config=config,
logger_creator=lambda _: UnifiedLogger(config, logdir))
# Use the connector server to generate experiences.
"input": (
lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, SERVER_PORT)
),
"observation_size": args.observation_size,
"action_size": args.action_size,
},
})
elif args.run == "PG":
agent = PGTrainer(
env="srv",
config={
"num_workers": 0,
"env_config": {
# Use the connector server to generate experiences.
"input": (
lambda ioctx: PolicyServerInput(ioctx, SERVER_ADDRESS, SERVER_PORT)
),
"observation_size": args.observation_size,
"action_size": args.action_size,
},
})
else:
raise ValueError("--run must be DQN or PG")
# Attempt to restore from checkpoint if possible.
if os.path.exists(args.checkpoint_file):
checkpoint_file = open(args.checkpoint_file).read()
print("Restoring from checkpoint path", checkpoint_file)
agent.restore(checkpoint_file)