def test5():
"""
Laplacian distribution is a continuous probability distribution
parameterized by loc and scale
:return:
"""
from torch.distributions.laplace import Laplace
dist = Laplace(torch.tensor([10.0]), torch.tensor([0.99]))
dist.sample() # >>> tensor([10.2167])
def addnoise(self, x):
"""Adds Laplacian noise to histogram of counts
Args:
counts[torch tensor]: Histogram counts
epsilon[integer]:Amount of Noise
Returns:
counts[torch tensor]: Noisy histogram of counts
"""
m = Laplace(torch.tensor([0.0]), torch.tensor([self.epsilon]))
count = x + m.sample()
return count
def __init__(self, train, holdout, tolerance=0.01/4, scale_factor=4, keep_log=True):
self.tolerance = tolerance
self.laplace_eps = Laplace(torch.tensor([0.0]), torch.tensor([2*self.tolerance]))
self.laplace_gamma = Laplace(torch.tensor([0.0]), torch.tensor([4*self.tolerance]))
self.laplace_eta = Laplace(torch.tensor([0.0]), torch.tensor([8*self.tolerance]))
self.train = train
self.holdout = holdout
self.T = 4*tolerance + self.noise(self.laplace_gamma)
# self.budget = ???
self.keep_log = keep_log
if keep_log:
self.log = pd.DataFrame(columns=['GlobStep', 'threshold', 'delta', 'phi_train', 'phi_holdout', 'estimate', 'overfit'])
def main():
flow = build_mera()
last_epoch = utils.get_last_checkpoint_step()
utils.load_checkpoint(last_epoch, flow)
flow.train(False)
shape = (16, args.nchannels, args.L, args.L)
prior_low = Laplace(torch.tensor(0.), torch.tensor(T_low / sqrt(2)))
z = prior_low.sample(shape)
prior_high = Laplace(torch.tensor(0.), torch.tensor(T_high / sqrt(2)))
z_high = prior_high.sample(shape)
k = 2**level_cutoff
z[:, :, ::k, ::k] = z_high[:, :, ::k, ::k]
z = z.to(args.device)
with torch.no_grad():
x, _ = flow.inverse(z)
samples = x.permute(0, 2, 3, 1).detach().cpu().numpy()
samples = 1 / (1 + np.exp(-samples))
fig, axes = plt.subplots(4, 4, figsize=(4, 4), sharex=True, sharey=True)
for i in range(4):
for j in range(4):
ax = axes[i, j]
ax.imshow(samples[j * 4 + i])
ax.axis('off')
plt.tight_layout()
plt.savefig('./mix_T.pdf', bbox_inches='tight')
def forward(self, obs, act):
x = torch.cat([obs, act], dim=-1)
hid_activation = []
# Hidden layers
for h_i in range(len(self.layers)):
x = self.layers[h_i](x)
# Store activation
if h_i % 2 == 1:
hid_activation.append(x)
# Output layer
mu = self.mu_layer(x)
log_std = self.log_std_layer(x)
log_std = torch.clamp(log_std, LOG_STD_MIN, LOG_STD_MAX)
if self.dist_type == 'Normal':
q_distribution = Normal(torch.squeeze(mu, -1),
torch.squeeze(torch.exp(log_std), -1))
elif self.dist_type == 'Laplace':
q_distribution = Laplace(torch.squeeze(mu, -1),
torch.squeeze(torch.exp(log_std), -1))
return torch.squeeze(mu, -1), torch.squeeze(
log_std, -1
), q_distribution, hid_activation # Critical to ensure q has right shape.
def sample_laplace(size, gpu=None):
m = Laplace(torch.tensor([0.0]), torch.tensor([1.0]))
y = m.sample(sample_shape=torch.Size([size[0], size[1], size[2]
])).float().squeeze(3)
y = y if gpu is None else y.cuda(gpu)
return y
# In[177]: # 0.20, 0.20, 0.20, 0.20,0.20 event probabilities # In[155]: # Laplace distribution parameterized by loc and ‘scale’. # In[157]: from torch.distributions.laplace import Laplace # In[161]: dist = Laplace(torch.tensor([10.0]), torch.tensor([0.990])) dist # In[162]: dist.sample() # In[163]: #Normal (Gaussian) distribution parameterized by loc and ‘scale’. # In[165]: from torch.distributions.normal import Normal # In[166]:
def addnoise(x):
m = Laplace(torch.tensor([0.0]), torch.tensor([0.5]))
count = x + m.sample().cuda()
return count
def compute_loss_q(data):
o, a, r, o2, d = data['obs'], data['act'], data['rew'], data[
'obs2'], data['done']
q_mu, q_log_std, q_distribution, q_hid_activation = ac.q(o, a)
q = q_mu
# Bellman backup for Q function
with torch.no_grad():
pi_targ, _ = ac_targ.pi(o2)
q_pi_mu_targ, q_pi_log_std_targ, q_pi_distribution_targ, _ = ac_targ.q(
o2, pi_targ)
q_pi_distribution_targ_entropy = q_pi_distribution_targ.entropy()
# Adaptive Gamma:
# Note: If not use adaptive_gamma, Ant-v2 tends to be overestimating.
if adaptive_gamma:
gamma = calculate_adaptive_gamma(
q_pi_distribution_targ_entropy,
beta=adap_gamma_beta,
gamma_min=adap_gamma_min,
gamma_max=adap_gamma_max,
tolerable_entropy=adap_gamma_tolerable_entropy)
else:
gamma = fixed_gamma
backup_mu = r + gamma * (1 - d) * q_pi_mu_targ
backup_log_std = (1 - d) * q_pi_log_std_targ
# Note: It's crucial to discount Standard Deviation too, otherwise high SD will be incorporated
# into online critic. We can also see this as an accumulated uncertainty.
backup_std = gamma * torch.exp(backup_log_std)
if dist_type == 'Normal':
backup_distribution = Normal(backup_mu, backup_std)
elif dist_type == 'Laplace':
backup_distribution = Laplace(backup_mu, backup_std)
backup = backup_mu
kl_loss = torch.diag(
torch.distributions.kl.kl_divergence(backup_distribution,
q_distribution)).mean()
q_error = (q - backup)
q_mse = (q_error**2).mean()
# loss_q = kl_loss + q_log_std.mean() # Not work for ReLU
# loss_q = kl_loss + torch.exp(q_log_std).mean() # Not good for ReLU, because this will make both Mu and Std increase.
# loss_q = kl_loss + 1e-5*q_distribution.entropy().mean()
# loss_q = kl_loss
# Note: KL-Divergence can be minimized just by increase standard deviation, if not optimize the MSE of mu.
loss_q = q_mse + 1e-2 * kl_loss
# loss_q = q_mse + 0.5*kl_loss + 0.01 * q_log_std.mean()
# loss_q = q_mse + torch.abs(q_error).mean()*kl_loss + q_log_std.mean()
# loss_q = q_mse
# import pdb; pdb.set_trace()
# Useful info for logging
if adaptive_gamma:
gamma_vals = gamma.detach().cpu().numpy()
else:
gamma_vals = gamma
loss_info = dict(
QMuVals=q_mu.detach().cpu().numpy(),
QStdVals=(gamma * torch.exp(q_log_std)).detach().cpu().numpy(),
QLogStdVals=q_log_std.detach().cpu().numpy(),
QHidActivation=torch.cat(q_hid_activation,
dim=1).detach().cpu().numpy(),
QMSE=q_mse.detach().cpu().numpy(),
QError=q_error.detach().cpu().numpy(),
QEntropy=q_distribution.entropy().detach().cpu().numpy(),
QKLDiv=kl_loss.detach().cpu().numpy(),
RVals=r.detach().cpu().numpy(),
Gamma=gamma_vals,
Entropy=q_pi_distribution_targ_entropy.detach().cpu().numpy())
return loss_q, loss_info
def sample(self, logits):
logits = safe_squeeze(logits, -1)
return Laplace(logits, 1.0).sample()