def __init__(self):
np.random.seed(1)
self.classifier = BayesClassifier(3)
self.classifier.add_class(Normal(), 0)
self.classifier.add_class(Normal(), 1)
self.classifier.add_class(Normal(), 2)
self.main()
def test_continuous(self):
data = [
12, 43, 41, 32, 34, 12, 43, 24, 21, 45, 34, 32, 10, 1, 2, 4, 49,
27, 11, 23
]
normal = Normal()
for num in data:
normal.add(num)
result = normal.get_parameters(recalculate=True)
self.assertAlmostEqual(25, result[0])
self.assertAlmostEqual(15.33829057929, result[1])
def test_remove_class_counter(self):
# Ensure that counter of added classes is correct after remove_class()
# function has been called
classifier = BayesClassifier(2)
classifier.add_class(Normal(), 1)
classifier.remove_class(1)
assert classifier.added_classes == 0
def test_remove_class(self):
# Ensure that distribution is removed from list after remove_class()
# function has been called
classifier = BayesClassifier(2)
classifier.add_class(Normal(), 1)
classifier.remove_class(1)
assert classifier.distributions[1] == None
def forward(self, action_embeds, feature, lengths):
''' Expects input state sequence as (batch, seq_len). Also requires a
list of lengths for dynamic batching. '''
x = action_embeds
ctx, _ = self.lstm(x)
# Att and Handle with the shape
batch_size, max_length, _ = ctx.size()
x, _ = self.attention_layer( # Attend to the feature map
ctx.contiguous().view(
-1, self.hidden_dim
), # (batch, length, hidden) --> (batch x length, hidden)
feature.view(
batch_size * max_length, -1, self.obs_dim
), # (batch, length, # of images, obs_dim) --> (batch x length, # of images, obs_dim)
)
x = x.view(batch_size, max_length, -1)
# Post LSTM layer
x, _ = self.post_lstm(x)
# need mean pooling for path_len
x = torch.mean(x, dim=1)
mean, log_var = self.mean_network(x), self.log_var_network(x)
log_var = torch.max(self.min_log_var, log_var)
dist = Normal(mean=mean, log_var=log_var)
return dist
def forward(self, z, s0):
'''
z: (bs, input_dim)
s0: initial obs (bs, view_dim, obs_dim) # view_dim = 36
'''
bs = z.size()[0]
self.init_state(bs)
if s0 is None:
s0 = Variable(torch.zeros(bs, self.obs_dim)) # init input
(bs, view_dim, obs_dim) = s0.shape
z = z.unsqueeze(1).repeat(1, view_dim, 1) # (bs, view_dim, latent_dim)
means, log_vars = [], []
x = s0
# NOTE: decode states by step 1 since we have fixed start point s0
for i in range(self.path_len):
# TODO: layernorm instead of concat: x=LayerNorm(A x) + LayerNorm(B z)
x = torch.cat([x, z], -1) # TODO: balance energy between x,z
x, attn = self.view_atten(x)
x = self.relu1(self.fc1(x)) # TODO: relu is necessary or not ?
x = x.unsqueeze(1) # bs, len, dim
mean, log_var = self.step(x)
x = mean.view(bs, self.view_num, -1)
means.append(mean)
log_vars.append(log_var)
means = torch.stack(means, 1).view(bs, -1)
log_vars = torch.stack(log_vars, 1).view(bs, -1)
dist = Normal(means, log_var=log_vars)
return dist
def __init__(self):
np.random.seed(1)
gamma = Gamma()
gamma.set_estimators(2)
self.classifier = BayesClassifier(2)
self.classifier.add_class(Normal(), 0)
self.classifier.add_class(gamma, 1)
self.main()
def act(self, x) : fx = self.features(x) xx = self.actor_final( fx ) meanxx, log_varxx = torch.chunk( xx, 2, dim=1) # scale the actions mean: unscaled = F.tanh(meanxx) self.mean = unscaled * self.action_scaler # log_var ; self.log_var = F.relu( log_varxx) #dist = Normal(self.mean, std=torch.sqrt(torch.exp(self.log_var)) ) self.dist = Normal(self.mean, log_var=self.log_var ) action = self.dist.sample() return action
def learn_distributions(self, properties=dict()):
for possible_world in self.__data:
for triple in possible_world:
prop_name = triple[1]
value = triple[2]
if properties is not None and prop_name in properties.keys():
prop = properties[prop_name]
prop.get_distribution().add(value)
else:
if type(value) is float:
new_prop = Property(prop_name, Normal())
new_prop.get_distribution().add(value)
properties[prop_name] = new_prop
else:
new_prop = Property(prop_name, Multinomial())
new_prop.get_distribution().add(value)
properties[prop_name] = new_prop
return properties
def __init__(self,
env,
tok,
obs_dim,
latent_dim,
loss_type="mse",
view_num=args.view_num,
path_len=2):
super(BaseVAE, self).__init__()
self.env = env
self.tok = tok
self.obs_dim = obs_dim
self.latent_dim = latent_dim
self.path_len = path_len
self.view_num = view_num
self.loss_type = loss_type
self.encoder = StateEncoder(obs_dim, latent_dim)
self.decoder = StateDecoder(obs_dim,
latent_dim,
view_num,
path_len=path_len)
self.policy = PolicyDecoder(obs_dim, latent_dim, view_num, path_len)
self.unit_n = Normal(Variable(torch.zeros(1, latent_dim)).cuda(),
log_var=Variable(torch.zeros(1,
latent_dim)).cuda())
self.encoder_optimizer = args.optimizer(self.encoder.parameters(),
lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(),
lr=args.lr)
self.policy_optimizer = args.optimizer(self.policy.parameters(),
lr=args.lr)
self.vae_loss_weight = args.vae_loss_weight
self.vae_kl_weight = args.vae_kl_weight
# self.bc_weight = 100
self.vae_bc_weight = args.vae_bc_weight
self.iter = None # training iteration indicator
class StochasticActorNN(nn.Module) :
def __init__(self,state_dim=3,action_dim=2,action_scaler=2.0,CNN={'use_cnn':False,'input_size':3},HER=True,actfn= lambda x : F.leaky_relu(x, 0.1) ) :
super(StochasticActorNN,self).__init__()
self.state_dim = state_dim
self.action_dim = action_dim
self.action_scaler = action_scaler
self.CNN = CNN
# dictionnary with :
# - 'input_size' : int
# - 'use_cnn' : bool
#
if self.CNN['use_cnn'] :
self.state_dim = self.CNN['input_size']
self.HER = HER
if self.HER :
self.state_dim *= 2
self.actfn = actfn
#Features :
if self.CNN['use_cnn'] :
self.conv1 = nn.Conv2d(self.state_dim,16, kernel_size=5, stride=2)
self.bn1 = nn.BatchNorm2d(16)
self.conv2 = nn.Conv2d(16,32, kernel_size=5, stride=2)
self.bn2 = nn.BatchNorm2d(32)
self.conv3 = nn.Conv2d(32,32, kernel_size=5, stride=2)
self.bn3 = nn.BatchNorm2d(32)
#self.featx = nn.Linear(448,self.nbr_actions)
#self.featx = nn.Linear(192,128)
self.featx = nn.Linear(2592,128)
else :
self.fc1 = nn.Linear(self.state_dim,400)
self.fc1.weight.data = init_weights(self.fc1.weight.data.size())
#self.bn1 = nn.BatchNorm1d(400)
self.fc2 = nn.Linear(400,300)
self.fc2.weight.data = init_weights(self.fc2.weight.data.size())
#self.bn2 = nn.BatchNorm1d(300)
#self.fc3 = nn.Linear(256,128)
#self.fc3.weight.data = init_weights(self.fc3.weight.data.size())
#self.bn3 = nn.BatchNorm1d(128)
#self.featx = nn.Linear(448,self.nbr_actions)
self.featx = nn.Linear(300,200)
self.featx.weight.data = init_weights(self.featx.weight.data.size())
# Actor network :
# normal distribution output : mean log_var
self.actor_final = nn.Linear(200,2*self.action_dim)
self.actor_final.weight.data.uniform_(-EPS,EPS)
def features(self,x) :
if self.CNN['use_cnn'] :
x1 = F.relu( self.bn1(self.conv1(x) ) )
x2 = F.relu( self.bn2(self.conv2(x1) ) )
x3 = F.relu( self.bn3(self.conv3(x2) ) )
x4 = x3.view( x3.size(0), -1)
#print(x4.size())
fx = F.relu( self.featx( x4) )
# batch x 128
else :
#x1 = F.relu( self.bn1(self.fc1(x) ) )
#x1 = F.leaky_relu( self.fc1(x), 0.1 )
x1 = self.actfn( self.fc1(x) )
#x2 = F.relu( self.bn2(self.fc2(x1) ) )
#x2 = F.leaky_relu( self.fc2(x1), 0.1 )
x2 = self.actfn( self.fc2(x1) )
#x3 = F.relu( self.fc3(x2) )
#x3 = F.relu( self.bn3(self.fc3(x2) ) )
#fx = F.relu( self.featx(x3) )
#fx = F.leaky_relu( self.featx(x2), 0.1 )
fx = self.actfn( self.featx( x2) )
#fx = F.relu( self.featx( x1) )
# batch x 128
return fx
def forward(self, x) :
fx = self.features(x)
xx = self.actor_final( fx )
meanxx, log_varxx = torch.chunk( xx, 2, dim=1)
# scale the actions mean:
unscaled = F.tanh(meanxx)
self.mean = unscaled * self.action_scaler
# log_var ;
self.log_var = F.relu( log_varxx)
#dist = Normal(self.mean, std=torch.sqrt(torch.exp(self.log_var)) )
self.dist = Normal(self.mean, log_var=self.log_var )
action = self.dist.sample()
return action
def randomCPUBoundJob():
dist = Normal(30, 10)
duration = random.randint(50, 200)
return Job(duration, dist)
def generate_good(good_name: str):
good = GoodKind(good_name)
good_index[good_name] = good
return good
food = generate_good("Food")
wood = generate_good("Wood")
basic_recipe_index = {}
def generate_basic_recipe(
labor: int,
good: GoodKind,
planet_variation: Distribution,
person_variation: Distribution,
labor_variation: Distribution,
required_goods=BagOfGoods(),
):
recipe = Recipe(labor, required_goods, planet_variation, person_variation,
labor_variation, good)
basic_recipe_index[good.name] = recipe
return recipe
basic_food_recipe = generate_basic_recipe(1, food, Normal(0.75, 0.5),
Normal(1, 0.3), Normal(1, 0.05))
basic_wood_recipe = generate_basic_recipe(1, wood, Normal(1, 0.5),
Normal(1, 0.2), Normal(1, 0.05))
class StochasticActorCriticNN(nn.Module) :
def __init__(self,state_dim=3,action_dim=2,action_scaler=2.0,CNN={'use_cnn':False,'input_size':3},dueling=True,HER=True) :
super(StochasticActorCriticNN,self).__init__()
self.state_dim = state_dim
self.action_dim = action_dim
self.action_scaler = action_scaler
self.dueling = dueling
self.CNN = CNN
# dictionnary with :
# - 'input_size' : int
# - 'use_cnn' : bool
#
if self.CNN['use_cnn'] :
self.state_dim = self.CNN['input_size']
self.HER = HER
if self.HER :
self.state_dim *= 2
#Features :
if self.CNN['use_cnn'] :
self.conv1 = nn.Conv2d(self.state_dim,16, kernel_size=5, stride=2)
self.bn1 = nn.BatchNorm2d(16)
self.conv2 = nn.Conv2d(16,32, kernel_size=5, stride=2)
self.bn2 = nn.BatchNorm2d(32)
self.conv3 = nn.Conv2d(32,32, kernel_size=5, stride=2)
self.bn3 = nn.BatchNorm2d(32)
#self.featx = nn.Linear(448,self.nbr_actions)
self.featx = nn.Linear(192,128)
else :
self.fc1 = nn.Linear(self.state_dim,512)
self.bn1 = nn.BatchNorm1d(512)
self.fc2 = nn.Linear(512,256)
self.bn2 = nn.BatchNorm1d(256)
#self.featx = nn.Linear(448,self.nbr_actions)
self.featx = nn.Linear(256,128)
self.featx.weight.data.uniform_(-EPS,EPS)
# Critic network :
## state value path :
if self.dueling :
self.critic_Vhead = nn.Linear(128,1)
else :
self.critic_Vhead = nn.Linear(128,64)
self.critic_Vhead.weight.data.uniform_(-EPS,EPS)
## action value path :
self.critic_afc1 = nn.Linear(self.action_dim,256)
self.critic_afc1.weight.data.uniform_(-EPS,EPS)
self.critic_afc2 = nn.Linear(256,128)
self.critic_afc2.weight.data.uniform_(-EPS,EPS)
if self.dueling :
self.critic_ahead = nn.Linear(256,128)
self.critic_ahead.weight.data.uniform_(-EPS,EPS)
else :
self.critic_ahead = nn.Linear(256,64)
self.critic_ahead.weight.data.uniform_(-EPS,EPS)
#linear layer, after the concatenation of ahead and vhead :
self.critic_final = nn.Linear(128,1)
self.critic_final.weight.data.uniform_(-EPS,EPS)
# Actor network :
self.actor_final = nn.Linear(128,2*self.action_dim)
self.actor_final.weight.data.uniform_(-EPS,EPS)
def features(self,x) :
if self.CNN['use_cnn'] :
x1 = F.relu( self.bn1(self.conv1(x) ) )
x2 = F.relu( self.bn2(self.conv2(x1) ) )
x3 = F.relu( self.bn3(self.conv3(x2) ) )
x4 = x3.view( x3.size(0), -1)
fx = F.relu( self.featx( x4) )
# batch x 128
else :
x1 = F.relu( self.bn1(self.fc1(x) ) )
x2 = F.relu( self.bn2(self.fc2(x1) ) )
fx = F.relu( self.featx( x2) )
# batch x 128
return fx
def evaluate(self, x,a) :
fx = self.features(x)
#V value :
self.Vvalue = v = self.critic_Vhead( fx )
a1 = F.relu( self.critic_afc1(a) )
a2 = F.relu( self.critic_afc2(a1) )
# batch x 128
afx = torch.cat([ fx, a2], dim=1)
# batch x 256
if self.dueling :
self.Advantage = advantage = self.critic_ahead(afx)
out = advantage + v
else :
advantage = self.critic_ahead(afx)
concat = torch.cat( [ v,advantage], dim=1)
out = self.critic_final(concat)
return out
def evaluateV(self, x) :
fx = self.features(x)
#V value :
self.Vvalue = v = self.critic_Vhead( fx )
return v
def act(self, x) :
fx = self.features(x)
xx = self.actor_final( fx )
meanxx, log_varxx = torch.chunk( xx, 2, dim=1)
# scale the actions mean:
unscaled = F.tanh(meanxx)
self.mean = unscaled * self.action_scaler
# log_var ;
self.log_var = F.relu( log_varxx)
#dist = Normal(self.mean, std=torch.sqrt(torch.exp(self.log_var)) )
self.dist = Normal(self.mean, log_var=self.log_var )
action = self.dist.sample()
return action
def test_exceeding_idx_add_class(self):
# Ensure that exception is raised when trying to add distribution
# on an exceeding idx in the add_class() function
classifier = BayesClassifier(4)
self.assertRaises(Exception, classifier.add_class, Normal(), 5)
def test_add_class_counter(self):
classifier = BayesClassifier(5)
classifier.add_class(Normal(), 0)
assert classifier.added_classes == 1
def test_add_class(self):
dist = Normal()
classifier = BayesClassifier(3)
classifier.add_class(dist, 0)
assert classifier.distributions[0] == dist