def __init__(self,
bot_name='RandomBot',
action_dim=2):
self.action_dim = action_dim
from opt_helpers import replay_buffer
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.bot_name = '../txts/'+bot_name
def __init__(self, bot_name='LSTMNet', input_dim=4, output_dim=2):
self.bot_name = bot_name
self.input_dim = input_dim
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.action_network = BaselineLSTMNet(input_dim=input_dim,
output_dim=output_dim,
is_value=False)
self.value_network = BaselineLSTMNet(input_dim=input_dim,
output_dim=output_dim,
is_value=True)
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters())
self.value_opt = torch.optim.RMSprop(self.value_network.parameters())
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.action_hidden_state = None
self.full_probs = None
self.value_hidden_state = None
self.new_action_hidden = None
self.new_value_hidden = None
self.reward_history = []
self.num_steps = 0
def __init__(self, bot_name='RandomProLoNet', input_dim=4, output_dim=2):
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
# self.replay_buffer = ReplayMemory(1000)
self.bot_name = bot_name
if input_dim == 4 and output_dim == 2:
self.action_network, self.value_network = init_random_cart_net()
elif input_dim == 8 and output_dim == 4:
self.action_network, self.value_network = init_random_lander_net()
elif input_dim == 194 and output_dim == 44:
self.action_network, self.value_network = init_random_sc_net()
elif input_dim == 32 and output_dim == 10:
self.action_network, self.value_network = init_random_micro_net()
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters())
self.value_opt = torch.optim.RMSprop(self.value_network.parameters())
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
distribution='one_hot',
bot_name='ProLoNet',
input_dim=4,
output_dim=2,
deepen_method='random',
deepen_criteria='entropy'):
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.bot_name = bot_name
if input_dim == 4 and output_dim == 2:
self.action_network, self.value_network = init_cart_nets(
distribution)
elif input_dim == 8 and output_dim == 4:
self.action_network, self.value_network = init_lander_nets(
distribution)
elif input_dim == 194 and output_dim == 44:
self.action_network, self.value_network = init_sc_nets(
distribution)
elif input_dim == 32 and output_dim == 10:
self.action_network, self.value_network = init_micro_net(
distribution)
self.deepen_method = deepen_method
self.deeper_action_network = add_level(self.action_network,
method=deepen_method)
self.deeper_value_network = add_level(self.value_network,
method=deepen_method)
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters())
self.value_opt = torch.optim.RMSprop(self.value_network.parameters())
self.deeper_actor_opt = torch.optim.RMSprop(
self.deeper_action_network.parameters())
self.deeper_value_opt = torch.optim.RMSprop(
self.deeper_value_network.parameters())
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = torch.Tensor([0])
self.last_deep_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.deeper_full_probs = None
self.reward_history = []
self.num_steps = 0
self.deepen_criteria = deepen_criteria
self.deepen_threshold = 350
self.times_deepened = 0
def __init__(self,
bot_name='DDT',
input_dim=4,
output_dim=2,
rule_list=False,
num_rules=4):
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.bot_name = bot_name
self.rule_list = rule_list
self.output_dim = output_dim
self.input_dim = input_dim
self.num_rules = num_rules
if rule_list:
self.bot_name += str(num_rules) + '_rules'
init_weights, init_comparators, init_leaves = init_rule_list(
num_rules, input_dim, output_dim)
else:
init_weights = None
init_comparators = None
init_leaves = num_rules
self.bot_name += str(num_rules) + '_leaves'
self.action_network = ProLoNet(input_dim=input_dim,
output_dim=output_dim,
weights=init_weights,
comparators=init_comparators,
leaves=init_leaves,
alpha=1,
is_value=False,
use_gpu=False)
self.value_network = ProLoNet(input_dim=input_dim,
output_dim=output_dim,
weights=init_weights,
comparators=init_comparators,
leaves=init_leaves,
alpha=1,
is_value=True,
use_gpu=False)
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True,
use_gpu=False)
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = None
self.last_deep_value_pred = [None] * output_dim
self.full_probs = None
self.deeper_full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
bot_name='DJINNAgent',
input_dim=4,
output_dim=2,
drop_prob=0.0):
self.bot_name = bot_name
self.input_dim = input_dim
self.output_dim = output_dim
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
if input_dim == 4:
self.env_name = 'cart'
elif input_dim == 6: # Fire Sim
self.env_name = 'fire'
elif input_dim == 8:
self.env_name = 'lunar'
elif input_dim == 37:
self.env_name = 'sc_micro'
elif input_dim > 100:
self.env_name = 'sc_macro'
tree_dict = DJINN_TREE_DATA[self.env_name]
action_param_dict = tree_to_nn_weights(input_dim, output_dim,
tree_dict)
# value_param_dict = tree_to_nn_weights(input_dim, 1, tree_dict)
self.action_network = PyDJINN(input_dim,
weights=action_param_dict['weights'],
biases=[],
drop_prob=drop_prob,
is_value=False)
self.value_network = PyDJINN(input_dim,
weights=action_param_dict['weights'],
biases=[],
drop_prob=drop_prob,
is_value=True)
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters(),
lr=5e-3)
self.value_opt = torch.optim.RMSprop(self.value_network.parameters(),
lr=5e-3)
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = torch.Tensor([0])
self.last_deep_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
distribution='one_hot',
bot_name='ShallowProLoNet',
input_dim=4,
output_dim=2,
adversarial=False):
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.bot_name = bot_name
self.adv_prob = 0.1
if input_dim == 4 and output_dim == 2:
if adversarial:
self.action_network, self.value_network = init_adversarial_net(adv_type='cart',
distribution_in=distribution,
adv_prob=self.adv_prob)
self.bot_name += '_adversarial' + str(self.adv_prob)
else:
self.action_network, self.value_network = init_cart_nets(distribution)
elif input_dim == 8 and output_dim == 4:
if adversarial:
self.action_network, self.value_network = init_adversarial_net(adv_type='lunar',
distribution_in=distribution)
self.bot_name += '_adversarial' + str(self.adv_prob)
else:
self.action_network, self.value_network = init_lander_nets(distribution)
elif input_dim == 194 and output_dim == 44:
if adversarial:
self.action_network, self.value_network = init_adversarial_net(adv_type='sc',
distribution_in=distribution)
self.bot_name += '_adversarial' + str(self.adv_prob)
else:
self.action_network, self.value_network = init_sc_nets(distribution)
elif input_dim == 32 and output_dim == 10:
if adversarial:
self.action_network, self.value_network = init_adversarial_net(adv_type='micro',
distribution_in=distribution)
self.bot_name += '_adversarial' + str(self.adv_prob)
else:
self.action_network, self.value_network = init_micro_net(distribution)
self.ppo = ppo_update.PPO([self.action_network, self.value_network], two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters())
self.value_opt = torch.optim.RMSprop(self.value_network.parameters())
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
bot_name='FCNet',
input_dim=4,
output_dim=2,
sl_init=False):
self.bot_name = bot_name
self.sl_init = sl_init
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.action_network = BaselineFCNet(input_dim=input_dim,
output_dim=output_dim,
is_value=False)
self.value_network = BaselineFCNet(input_dim=input_dim,
output_dim=output_dim,
is_value=True)
if self.sl_init:
if input_dim == 4:
self.teacher = CartPoleHeuristic()
self.action_loss_threshold = 250
elif input_dim == 8:
self.teacher = LunarHeuristic()
self.action_loss_threshold = 350
elif input_dim == 32:
self.teacher = StarCraftMicroHeuristic()
self.action_loss_threshold = 500
elif input_dim > 100:
self.teacher = StarCraftMacroHeuristic()
self.action_loss_threshold = 1000
self.bot_name += '_SLtoRL_'
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters())
self.value_opt = torch.optim.RMSprop(self.value_network.parameters())
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = torch.Tensor([0])
self.last_deep_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
distribution='one_hot',
bot_name='ProLoNet',
input_dim=4,
output_dim=2,
use_gpu=False,
vectorized=False,
randomized=False,
adversarial=False,
deepen=True,
epsilon=0.9,
epsilon_decay=0.95,
epsilon_min=0.05,
deterministic=False):
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.bot_name = bot_name
self.use_gpu = use_gpu
self.vectorized = vectorized
self.randomized = randomized
self.adversarial = adversarial
self.deepen = deepen
self.output_dim = output_dim
self.input_dim = input_dim
self.adv_prob = .05
self.epsilon = epsilon
self.epsilon_decay = epsilon_decay
self.epsilon_min = epsilon_min
self.deterministic = deterministic
if vectorized:
self.bot_name += '_vect'
if randomized:
self.bot_name += '_rand'
if use_gpu:
self.bot_name += '_gpu'
if deepen:
self.bot_name += '_deepening'
if input_dim == 4 and output_dim == 2: # CartPole
self.action_network, self.value_network = init_cart_nets(
distribution, use_gpu, vectorized, randomized)
if adversarial:
self.action_network, self.value_network = init_adversarial_net(
adv_type='cart',
distribution_in=distribution,
adv_prob=self.adv_prob)
self.bot_name += '_adversarial' + str(self.adv_prob)
elif input_dim == 8 and output_dim == 4: # Lunar Lander
self.action_network, self.value_network = init_lander_nets(
distribution, use_gpu, vectorized, randomized)
if adversarial:
self.action_network, self.value_network = init_adversarial_net(
adv_type='lunar',
distribution_in=distribution,
adv_prob=self.adv_prob)
self.bot_name += '_adversarial' + str(self.adv_prob)
elif input_dim == 194 and output_dim == 44: # SC Macro
self.action_network, self.value_network = init_sc_nets(
distribution, use_gpu, vectorized, randomized)
elif input_dim == 37 and output_dim == 10: # SC Micro
self.action_network, self.value_network = init_micro_net(
distribution, use_gpu, vectorized, randomized)
if adversarial:
self.action_network, self.value_network = init_adversarial_net(
adv_type='micro',
distribution_in=distribution,
adv_prob=self.adv_prob)
self.bot_name += '_adversarial' + str(self.adv_prob)
elif input_dim == 6 and output_dim == 5: # Fire Sim
self.action_network, self.value_network = init_fire_nets(
distribution, use_gpu, vectorized, randomized,
bot_name.split('_')[0])
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True,
use_gpu=use_gpu)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters(),
lr=1e-5)
self.value_opt = torch.optim.RMSprop(self.value_network.parameters(),
lr=1e-5)
if self.deepen:
self.deeper_action_network = add_level(self.action_network,
use_gpu=use_gpu)
self.deeper_value_network = add_level(self.value_network,
use_gpu=use_gpu)
self.deeper_actor_opt = torch.optim.RMSprop(
self.deeper_action_network.parameters())
self.deeper_value_opt = torch.optim.RMSprop(
self.deeper_value_network.parameters())
else:
self.deeper_value_network = None
self.deeper_action_network = None
self.deeper_actor_opt = None
self.deeper_value_opt = None
self.num_times_deepened = 0
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = None
self.last_deep_value_pred = [None] * output_dim
self.full_probs = None
self.deeper_full_probs = None
self.reward_history = []
self.num_steps = 0
def __init__(self,
bot_name='Lunar_Heuristic'):
self.bot_name = '../txts/'+bot_name
from opt_helpers import replay_buffer
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
def __init__(self,
bot_name='CartPoleHeuristic',
params=None):
self.bot_name = '../txts/'+bot_name
from opt_helpers import replay_buffer
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
def __init__(self,
bot_name='FCNet',
input_dim=4,
output_dim=2,
sl_init=False,
num_hidden=1):
self.bot_name = bot_name + str(num_hidden) + '_hid'
self.sl_init = sl_init
self.input_dim = input_dim
self.output_dim = output_dim
self.num_hidden = num_hidden
self.replay_buffer = replay_buffer.ReplayBufferSingleAgent()
self.action_network = BaselineFCNet(input_dim=input_dim,
output_dim=output_dim,
is_value=False,
hidden_layers=num_hidden)
self.value_network = BaselineFCNet(input_dim=input_dim,
output_dim=output_dim,
is_value=True,
hidden_layers=num_hidden)
if self.sl_init:
if input_dim == 4:
self.teacher = CartPoleHeuristic()
self.action_loss_threshold = 25
elif input_dim == 6: # Fire Sim
self.teacher = DeepProLoNet(
distribution='one_hot',
input_dim=input_dim,
output_dim=output_dim,
use_gpu=False,
vectorized=False,
randomized=False,
adversarial=False,
deepen=False,
deterministic=True,
)
self.action_loss_threshold = 50
elif input_dim == 12: # Build Marines
self.teacher = DeepProLoNet(
distribution='one_hot',
input_dim=input_dim,
output_dim=output_dim,
use_gpu=False,
vectorized=False,
randomized=False,
adversarial=False,
deepen=False,
deterministic=True,
)
self.action_loss_threshold = 50
elif input_dim == 8:
self.teacher = LunarHeuristic()
self.action_loss_threshold = 35
elif input_dim == 28:
self.teacher = DeepProLoNet(
distribution='one_hot',
input_dim=input_dim,
output_dim=output_dim,
use_gpu=False,
vectorized=False,
randomized=False,
adversarial=False,
deepen=False,
deterministic=True,
)
self.teacher.load()
self.action_loss_threshold = 50
elif input_dim == 37:
self.teacher = StarCraftMicroHeuristic()
self.action_loss_threshold = 50
elif input_dim > 100:
self.teacher = StarCraftMacroHeuristic()
self.action_loss_threshold = 1000
self.bot_name += '_SLtoRL_'
self.ppo = ppo_update.PPO([self.action_network, self.value_network],
two_nets=True)
self.actor_opt = torch.optim.RMSprop(self.action_network.parameters(),
lr=5e-3)
self.value_opt = torch.optim.RMSprop(self.value_network.parameters(),
lr=5e-3)
# self.ppo.actor_opt = self.actor_opt
# self.ppo.critic_opt = self.value_opt
self.last_state = [0, 0, 0, 0]
self.last_action = 0
self.last_action_probs = torch.Tensor([0])
self.last_value_pred = torch.Tensor([[0, 0]])
self.last_deep_action_probs = torch.Tensor([0])
self.last_deep_value_pred = torch.Tensor([[0, 0]])
self.full_probs = None
self.reward_history = []
self.num_steps = 0