def __call__(self, module, features):
statistics = self.get_statistics(features)
self.statistics = statistics
if self.mode == 'store':
self.stored[module] = statistics.detach()
elif self.mode == 'match':
if statistics.ndimension() == 2:
if self.method == 'maximize':
self.losses[module] = - statistics[0, self.map_index]
else:
self.losses[module] = torch.abs(300 - statistics[0, self.map_index])
else:
ws = self.window_size
t = statistics.detach() * 0
s_cc = statistics[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
t_cc = t[:1, :, t.shape[2] // 2 - ws:t.shape[2] // 2 + ws, t.shape[3] // 2 - ws:t.shape[3] // 2 + ws] #* 1.0
t_cc[:, self.map_index,...] = 1
if self.method == 'maximize':
self.losses[module] = -(s_cc * t_cc.contiguous()).sum()
else:
self.losses[module] = torch.abs(200 -(s_cc * t_cc.contiguous())).sum()
def kurtosis_score(x, dim=0):
'''Test whether a dataset has normal kurtosis.
This function tests the null hypothesis that the kurtosis
of the population from which the sample was drawn is that
of the normal distribution: ``kurtosis = 3(n-1)/(n+1)``.
ripoff from: `scipy.stats.kurtosistest`.
Args:
a: Array of the sample data
axis: Axis along which to compute test. Default is 0. If None,
compute over the whole array `a`.
Returns:
statistic: The computed z-score for this test.
p-value: A 2-sided chi squared probability for the hypothesis test.
'''
x, n, dim = _x_n_dim(x, dim)
if n < 20:
raise ValueError(
"Number of elements has to be >= 20 to compute kurtosis")
b2 = (x**4).mean(dim) / (x**2).mean(dim)**2
E = 3.0 * (n - 1) / (n + 1)
varb2 = 24.0 * n * (n - 2) * (n - 3) / ((n + 1)**2 * (n + 3) * (n + 5))
x = (b2 - E) / math.sqrt(varb2)
sqrtbeta1 = 6.0 * (n * n - 5 * n + 2) / ((n + 7) * (n + 9)) *\
math.sqrt((6.0 * (n + 3) * (n + 5)) / (n * (n - 2) * (n - 3)))
A = 6.0 + 8.0 / sqrtbeta1 * \
(2.0 / sqrtbeta1 + math.sqrt(1 + 4.0 / (sqrtbeta1**2)))
term1 = 1 - 2 / (9.0 * A)
denom = 1 + x * math.sqrt(2 / (A - 4.0))
term2 = torch.sign(denom) * torch.pow((1 - 2.0 / A) /
torch.abs(denom), 1 / 3.0)
Z = (term1 - term2) / math.sqrt(2 / (9.0 * A))
return Z, 1 + torch.erf(-math.sqrt(0.5) * torch.abs(Z))
def forward(self, agent_qs, states):
"""Forward pass for the mixer.
Arguments:
agent_qs: Tensor of shape [B, T, n_agents, n_actions]
states: Tensor of shape [B, T, state_dim]
"""
bs = agent_qs.size(0)
states = states.reshape(-1, self.state_dim)
agent_qs = agent_qs.view(-1, 1, self.n_agents)
# First layer
w1 = th.abs(self.hyper_w_1(states))
b1 = self.hyper_b_1(states)
w1 = w1.view(-1, self.n_agents, self.embed_dim)
b1 = b1.view(-1, 1, self.embed_dim)
hidden = F.elu(th.bmm(agent_qs, w1) + b1)
# Second layer
w_final = th.abs(self.hyper_w_final(states))
w_final = w_final.view(-1, self.embed_dim, 1)
# State-dependent bias
v = self.V(states).view(-1, 1, 1)
# Compute final output
y = th.bmm(hidden, w_final) + v
# Reshape and return
q_tot = y.view(bs, -1, 1)
return q_tot
def test_MultivariateNormalQMCEngineDegenerate(self, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
for dtype in (torch.float, torch.double):
# X, Y iid standard Normal and Z = X + Y, random vector (X, Y, Z)
mean = torch.zeros(3, device=device, dtype=dtype)
cov = torch.tensor(
[[1, 0, 1], [0, 1, 1], [1, 1, 2]], device=device, dtype=dtype
)
engine = MultivariateNormalQMCEngine(mean=mean, cov=cov, seed=12345)
samples = engine.draw(n=2000)
self.assertEqual(samples.dtype, dtype)
self.assertEqual(samples.device.type, device.type)
self.assertTrue(torch.all(torch.abs(samples.mean(dim=0)) < 1e-2))
self.assertTrue(torch.abs(torch.std(samples[:, 0]) - 1) < 1e-2)
self.assertTrue(torch.abs(torch.std(samples[:, 1]) - 1) < 1e-2)
self.assertTrue(torch.abs(torch.std(samples[:, 2]) - math.sqrt(2)) < 1e-2)
for i in (0, 1, 2):
_, pval = shapiro(samples[:, i].cpu().numpy())
self.assertGreater(pval, 0.9)
cov = np.cov(samples.cpu().numpy().transpose())
self.assertLess(np.abs(cov[0, 1]), 1e-2)
self.assertLess(np.abs(cov[0, 2] - 1), 1e-2)
# check to see if X + Y = Z almost exactly
self.assertTrue(
torch.all(
torch.abs(samples[:, 0] + samples[:, 1] - samples[:, 2]) < 1e-5
)
)
def simplax(surrogate, x, logits, mixtureweights, k=1):
B = logits.shape[0]
probs = torch.softmax(logits, dim=1)
cat = RelaxedOneHotCategorical(probs=probs, temperature=torch.tensor([1.]).cuda())
outputs = {}
net_loss = 0
surr_loss = 0
for jj in range(k):
cluster_S = cat.rsample()
cluster_H = H(cluster_S)
logq = cat.log_prob(cluster_S.detach()).view(B,1)
logpx_given_z = logprob_undercomponent(x, component=cluster_H)
logpz = torch.log(mixtureweights[cluster_H]).view(B,1)
logpxz = logpx_given_z + logpz #[B,1]
f = logpxz - logq - 1.
surr_input = torch.cat([cluster_S, x, logits], dim=1) #[B,21]
surr_pred = surrogate.net(surr_input)
net_loss += - torch.mean((f.detach() - surr_pred.detach()) * logq + surr_pred)
# surr_loss += torch.mean(torch.abs(f.detach()-1.-surr_pred))
# grad_logq = torch.mean( torch.autograd.grad([torch.mean(logq)], [logits], create_graph=True, retain_graph=True)[0], dim=1, keepdim=True)
# grad_surr = torch.mean( torch.autograd.grad([torch.mean(surr_pred)], [logits], create_graph=True, retain_graph=True)[0], dim=1, keepdim=True)
grad_logq = torch.autograd.grad([torch.mean(logq)], [logits], create_graph=True, retain_graph=True)[0]
grad_surr = torch.autograd.grad([torch.mean(surr_pred)], [logits], create_graph=True, retain_graph=True)[0]
surr_loss = torch.mean(((f.detach() - surr_pred) * grad_logq + grad_surr)**2)
surr_dif = torch.mean(torch.abs(f.detach() - surr_pred))
# surr_loss = torch.mean(torch.abs(f.detach() - surr_pred))
grad_path = torch.autograd.grad([torch.mean(surr_pred)], [logits], create_graph=True, retain_graph=True)[0]
grad_score = torch.autograd.grad([torch.mean((f.detach() - surr_pred.detach()) * logq)], [logits], create_graph=True, retain_graph=True)[0]
grad_path = torch.mean(torch.abs(grad_path))
grad_score = torch.mean(torch.abs(grad_score))
net_loss = net_loss/ k
surr_loss = surr_loss/ k
outputs['net_loss'] = net_loss
outputs['f'] = f
outputs['logpx_given_z'] = logpx_given_z
outputs['logpz'] = logpz
outputs['logq'] = logq
outputs['surr_loss'] = surr_loss
outputs['surr_dif'] = surr_dif
outputs['grad_path'] = grad_path
outputs['grad_score'] = grad_score
return outputs #net_loss, f, logpx_given_z, logpz, logq, surr_loss, surr_dif, grad_path, grad_score
def argwhere_nonzero(layer, batchnorm=False):
indices=[]
# for batchnorms we want to do the opposite
if batchnorm:
for idx,w in enumerate(layer):
if torch.sum(torch.abs(w)).data.cpu().numpy() == 0.:
indices.append(idx)
else:
for idx,w in enumerate(layer):
if torch.sum(torch.abs(w)).data.cpu().numpy() != 0.:
indices.append(idx)
return indices
def test_NormalQMCEngineShapiroInvTransform(self):
engine = NormalQMCEngine(d=2, seed=12345, inv_transform=True)
samples = engine.draw(n=250)
self.assertEqual(samples.dtype, torch.float)
self.assertTrue(torch.all(torch.abs(samples.mean(dim=0)) < 1e-2))
self.assertTrue(torch.all(torch.abs(samples.std(dim=0) - 1) < 1e-2))
# perform Shapiro-Wilk test for normality
for i in (0, 1):
_, pval = shapiro(samples[:, i])
self.assertGreater(pval, 0.9)
# make sure samples are uncorrelated
cov = np.cov(samples.numpy().transpose())
self.assertLess(np.abs(cov[0, 1]), 1e-2)
def forward(self, x1, x2):
out1 = self.forward_one(x1)
out2 = self.forward_one(x2)
dis = torch.abs(out1 - out2)
out = self.out(dis)
# return self.sigmoid(out)
return out
def _mu_law(self, x):
m = self._variable(torch.FloatTensor(1))
m[:] = self.n_categories + 1
s = torch.sign(x)
x = torch.abs(x)
x = s * (torch.log(1 + (self.n_categories * x)) / torch.log(m))
return x
def pairwise_distance(x1, x2, p=2, eps=1e-6):
r"""
Computes the batchwise pairwise distance between vectors v1,v2:
.. math ::
\Vert x \Vert _p := \left( \sum_{i=1}^n \vert x_i \vert ^ p \right) ^ {1/p}
Args:
x1: first input tensor
x2: second input tensor
p: the norm degree. Default: 2
eps (float, optional): Small value to avoid division by zero. Default: 1e-6
Shape:
- Input: :math:`(N, D)` where `D = vector dimension`
- Output: :math:`(N, 1)`
Example::
>>> input1 = autograd.Variable(torch.randn(100, 128))
>>> input2 = autograd.Variable(torch.randn(100, 128))
>>> output = F.pairwise_distance(input1, input2, p=2)
>>> output.backward()
"""
assert x1.size() == x2.size(), "Input sizes must be equal."
assert x1.dim() == 2, "Input must be a 2D matrix."
diff = torch.abs(x1 - x2)
out = torch.pow(diff + eps, p).sum(dim=1, keepdim=True)
return torch.pow(out, 1. / p)
def forward(self, frame, policies):
# x: [B,2,84,84]
self.B = frame.size()[0]
#Predict mask
pre_mask = self.predict_mask_nosigmoid(frame)
mask = F.sigmoid(pre_mask)
masked_frame = frame * mask
kls = []
for i in range(len(policies)):
policy = policies[i]
log_dist_mask = policy.action_logdist(masked_frame)
log_dist_true = policy.action_logdist(frame)
action_dist_kl = torch.sum((log_dist_true - log_dist_mask)*torch.exp(log_dist_true), dim=1) #[B]
action_dist_kl = torch.mean(action_dist_kl) # * 1000
kls.append(action_dist_kl)
kls = torch.stack(kls) #[policies, B]
action_dist_kl = torch.mean(action_dist_kl) #[1] #over batch and over policies
pre_mask = pre_mask.view(self.B, -1)
mask_cost = torch.abs(pre_mask + 20)
# mask_sum = torch.mean(torch.sum(mask_cost, dim=1)) * .00001
# mask_cost = torch.mean(mask_cost) * .00001
mask_cost = torch.mean(mask_cost) * .01
loss = action_dist_kl + mask_cost
return loss, action_dist_kl, mask_cost
def test_train(self):
self._metric.train()
calls = [[torch.FloatTensor([0.0]), torch.LongTensor([0])],
[torch.FloatTensor([0.0, 0.1, 0.2, 0.3]), torch.LongTensor([0, 1, 2, 3])]]
for i in range(len(self._states)):
self._metric.process(self._states[i])
self.assertEqual(2, len(self._metric_function.call_args_list))
for i in range(len(self._metric_function.call_args_list)):
self.assertTrue(torch.eq(self._metric_function.call_args_list[i][0][0], calls[i][0]).all)
self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[i][0][1], -calls[i][1])), 1e-12).all)
self._metric_function.reset_mock()
self._metric.process_final({})
self._metric_function.assert_called_once()
self.assertTrue(torch.eq(self._metric_function.call_args_list[0][0][1], torch.LongTensor([0, 1, 2, 3, 4])).all)
self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[0][0][0], -torch.FloatTensor([0.0, 0.1, 0.2, 0.3, 0.4]))), 1e-12).all)
def skewness_score(x, dim=0):
'''Test whether the skew is different from the normal distribution.
This function tests the null hypothesis that the skewness of
the population that the sample was drawn from is the same
as that of a corresponding normal distribution.
ripoff from: `scipy.stats.skewtest`.
Args:
a: Array of the sample data
axis: Axis along which to compute test. Default is 0. If None,
compute over the whole array `a`.
Returns:
statistic: The computed z-score for this test.
p-value: A 2-sided chi squared probability for the hypothesis test.
'''
x, n, dim = _x_n_dim(x, dim)
b2 = (x**3).mean(dim) / (x**2).mean(dim)**1.5
y = b2 * math.sqrt(((n + 1) * (n + 3)) / (6.0 * (n - 2)))
beta2 = 3.0 * (n**2 + 27 * n - 70) * (n + 1) * (n + 3) /\
((n - 2.0) * (n + 5) * (n + 7) * (n + 9))
W2 = -1.0 + math.sqrt(2 * (beta2 - 1))
delta = 1.0 / math.sqrt(0.5 * math.log(W2))
alpha = math.sqrt(2.0 / (W2 - 1))
y[y == 0] = 1
yalpha = y / alpha
Z = delta * torch.log(yalpha + torch.sqrt(yalpha**2 + 1))
return Z, 1 + torch.erf(-math.sqrt(0.5) * torch.abs(Z))
def apply_global_reward(self, rewards: torch.Tensor, next_iteration: int):
std_dev = torch.std(rewards)
if torch.abs(std_dev) > 1e-6:
normalized_rewards = (rewards - torch.mean(rewards)) / std_dev
for parent_tensor in self.parent_tensors.values():
parent_tensor.grad.zero_()
for i, individual in enumerate(self.population_tensors):
for tensor_name, parent_tensor in self.parent_tensors.items():
individual_tensor = individual[tensor_name]
# Subtract the parent to get the gradient estimate
individual_tensor.sub_(parent_tensor)
# Amplify the gradient by the reward
individual_tensor.mul_(normalized_rewards[i])
# Divide by a normalizing constant
individual_tensor.div_(
self.es_params.population_size
* self.es_params.mutation_power
* -1
)
parent_tensor.grad += individual_tensor
self.optimizer.step()
self.populate_children(next_iteration)
def evalAccuracy(model, device, args, outPutType, test_X, test_Y, validEval = False) :
model_training_orig = model.training
setModelMode(model, False)
N_test = len(test_Y)*len(test_Y[0])
totals = 0
for b_data, b_labels in zip(test_X, test_Y):
b_data = torch.from_numpy(b_data).to(device)
b_labels = torch.from_numpy(b_labels).to(device)
b_labels = b_labels.view(b_labels.shape[0],-1 ) # make it the same shape as output
yhat = model(b_data) # need to compute the Yhat again, as this is the yhat AFTER updating the weights, not before as in 'learn()' function
b_data = None
# depending on if we are in a classification or regression problem, we evaluate performance differently
if outPutType == OUT_REGRESSION :
currentRate = torch_pearsonr( yhat.view(-1) , b_labels.view(-1))**2
N_test = len(test_Y) # as we are testing correlation, the N should refer to the number of batches, and NOT the total number of observations
elif outPutType == OUT_MULTICLASS :
currentRate = calc_Accuracy(yhat,b_labels ) # calculate accuracy, this is ROUNDED
else : # mean absolute error
currentRate = -torch.mean(torch.abs(yhat - b_labels)) # negative as the rest of the metrics are accuracy, IE the greater the error, the lower the accuracy
N_test = len(test_Y) # as we are testing correlation, the N should refer to the number of batches, and NOT the total number of observations
currentRate = float(currentRate.detach().cpu().numpy() ) # need to move it back to CPU
totals = totals +currentRate # sum in all minibatches
accuracy = round( float(totals)/N_test,5)
setModelMode(model, model_training_orig)
return(accuracy)
def test_autograd_closure(self):
x = Variable(torch.Tensor([0.4]), requires_grad=True)
y = Variable(torch.Tensor([0.7]), requires_grad=True)
trace = torch._C._tracer_enter((x, y), 1)
z = torch.sigmoid(x * (x + y))
w = torch.abs(x * x * x + y) + Variable(torch.ones(1))
torch._C._tracer_exit((z, w))
torch._C._jit_pass_lint(trace)
(z * w).backward()
torch._C._jit_pass_dce(trace)
torch._C._jit_pass_lint(trace)
x_grad = x.grad.data.clone()
x.grad.data.zero_()
function = torch._C._jit_createAutogradClosure(trace)
torch._C._jit_pass_lint(trace)
z2, w2 = function()(x, y)
(z2 * w2).backward()
self.assertEqual(z, z2)
self.assertEqual(w, w2)
self.assertEqual(x.grad.data, x_grad)
def test_regularization(self):
penalty = self.model.get_regularization_penalty().data
assert (penalty > 0).all()
penalty2 = 0
# Config specifies penalty as
# "regularizer": [
# ["weight$", {"type": "l2", "alpha": 10}],
# ["bias$", {"type": "l1", "alpha": 5}]
# ]
for name, parameter in self.model.named_parameters():
if name.endswith("weight"):
weight_penalty = 10 * torch.sum(torch.pow(parameter, 2))
penalty2 += weight_penalty
elif name.endswith("bias"):
bias_penalty = 5 * torch.sum(torch.abs(parameter))
penalty2 += bias_penalty
assert (penalty == penalty2.data).all()
# You get a RuntimeError if you call `model.forward` twice on the same inputs.
# The data and config are such that the whole dataset is one batch.
training_batch = next(self.iterator(self.instances, num_epochs=1))
validation_batch = next(self.iterator(self.instances, num_epochs=1))
training_loss = self.trainer._batch_loss(training_batch, for_training=True).data
validation_loss = self.trainer._batch_loss(validation_batch, for_training=False).data
# Training loss should have the regularization penalty, but validation loss should not.
assert (training_loss != validation_loss).all()
# Training loss should equal the validation loss plus the penalty.
penalized = validation_loss + penalty
assert (training_loss == penalized).all()
def evaluate_post_training(self, edp: EvaluationDataPage) -> CpeDetails:
cpe_details = CpeDetails()
self.score_cpe("Reward", edp, cpe_details.reward_estimates)
if (
self.metrics_to_score is not None
and edp.logged_metrics is not None
and self.action_names is not None
):
for i, metric in enumerate(self.metrics_to_score):
logger.info(
"--------- Running CPE on metric: {} ---------".format(metric)
)
metric_reward_edp = edp.set_metric_as_reward(i, len(self.action_names))
cpe_details.metric_estimates[metric] = CpeEstimateSet()
self.score_cpe(
metric, metric_reward_edp, cpe_details.metric_estimates[metric]
)
# Compute MC Loss on Aggregate Reward
cpe_details.mc_loss = float(
torch.mean(torch.abs(edp.logged_values - edp.model_values))
)
return cpe_details
def test_mu_law_companding(self):
sig = self.sig.clone()
quantization_channels = 256
sig = self.sig.numpy()
sig = sig / np.abs(sig).max()
self.assertTrue(sig.min() >= -1. and sig.max() <= 1.)
sig_mu = transforms.MuLawEncoding(quantization_channels)(sig)
self.assertTrue(sig_mu.min() >= 0. and sig.max() <= quantization_channels)
sig_exp = transforms.MuLawExpanding(quantization_channels)(sig_mu)
self.assertTrue(sig_exp.min() >= -1. and sig_exp.max() <= 1.)
sig = self.sig.clone()
sig = sig / torch.abs(sig).max()
self.assertTrue(sig.min() >= -1. and sig.max() <= 1.)
sig_mu = transforms.MuLawEncoding(quantization_channels)(sig)
self.assertTrue(sig_mu.min() >= 0. and sig.max() <= quantization_channels)
sig_exp = transforms.MuLawExpanding(quantization_channels)(sig_mu)
self.assertTrue(sig_exp.min() >= -1. and sig_exp.max() <= 1.)
repr_test = transforms.MuLawEncoding(quantization_channels)
repr_test.__repr__()
repr_test = transforms.MuLawExpanding(quantization_channels)
repr_test.__repr__()
def get_loss(self, y_pred, y_true, X=None, training=False):
y_true = to_tensor(y_true, device='cpu')
loss_a = torch.abs(y_true.float() - y_pred[:, 1]).mean()
loss_b = ((y_true.float() - y_pred[:, 1]) ** 2).mean()
if training:
self.history.record_batch('loss_a', to_numpy(loss_a))
self.history.record_batch('loss_b', to_numpy(loss_b))
return loss_a + loss_b
def forward(self, x, y, z):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2(x), 2))
x = x.view(-1, 1600)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return th.abs(10 - x)
def sparsify(model, sparsity_level=50.):
for name, param in model.named_parameters():
if 'weight' in name:
threshold = calculate_threshold(param.data, sparsity_level)
mask = torch.gt(torch.abs(param.data), threshold).float()
param.data = param.data * mask
return model
def __call__(self, y_pred, y_true=None):
"""
y_pred should be two projections
"""
covar_mat = th.abs(th_matrixcorr(y_pred[0].data, y_pred[1].data))
self.corr_sum += th.trace(covar_mat)
self.total_count += covar_mat.size(0)
return self.corr_sum / self.total_count
def forward(self, s1, s2):
# s1 : (s1, s1_len)
u = self.encoder(s1)
v = self.encoder(s2)
features = torch.cat((u, v, torch.abs(u-v), u*v), 1)
output = self.classifier(features)
return output
def forward(self, input):
# Hack: Force noise vectors to be function of input so they are put into
# predict_net and not init_net when tracing with ONNX
epsilon_input = torch.randn(1, input.size()[1], device=input.device)
epsilon_output = torch.randn(
self.out_dimension - input.size()[1] + input.size()[1],
1,
device=input.device,
)
epsilon_in = torch.sign(epsilon_input) * torch.sqrt(torch.abs(epsilon_input))
epsilon_out = torch.sign(epsilon_output) * torch.sqrt(torch.abs(epsilon_output))
# Add noise to bias and weights
noise = torch.mul(epsilon_in, epsilon_out)
bias = self.bias + self.sigma_bias * epsilon_out.t()
weight = self.weight + self.sigma_weight * noise
return input.matmul(weight.t()) + bias
def log_abs_det_jacobian(self, x, y):
result = torch.abs(self.scale).log()
shape = x.shape
if self.event_dim:
result_size = result.size()[:-self.event_dim] + (-1,)
result = result.view(result_size).sum(-1)
shape = shape[:-self.event_dim]
return result.expand(shape)
def smooth_L1(pred,targets,alpha_in,alpha_out,beta=1.0):
x=(pred-targets)*alpha_in
xabs=torch.abs(x)
y1=0.5*x**2/beta
y2=xabs-0.5*beta
case1=torch.le(xabs,beta).float()
case2=1-case1
return torch.sum((y1*case1+y2*case2)*alpha_out)/pred.size(0)
def forward(self, lvec, rvec):
mult_dist = torch.mul(lvec, rvec)
abs_dist = torch.abs(torch.add(lvec, -rvec))
vec_dist = torch.cat((mult_dist, abs_dist), 1)
out = F.sigmoid(self.wh(vec_dist))
out = F.log_softmax(self.wp(out))
return out
def build_Phi(self, e, mask):
e_rows = e.unsqueeze(1).expand(self.batch_size, self.n, self.n)
e_cols = e.unsqueeze(2).expand(self.batch_size, self.n, self.n)
Phi = Variable(torch.abs(e_rows - e_cols).data == 0).type(dtype)
# mask attention matrix
mask_rows = mask.unsqueeze(2).expand_as(Phi)
mask_cols = mask.unsqueeze(1).expand_as(Phi)
Phi = Phi * mask_rows * mask_cols
return Phi
def addOrthoRegularizer(loss,model, regParam, targetLayers) :
for i in range( len(targetLayers) ) :
layerParams = model[targetLayers[i]].named_parameters()
for param in layerParams: # dont regularize bias params
if 'bias' not in param[0]:
W = param[1].t()
WTW = torch.mm( W.t(), W)
C = ( regParam * 0.5) * torch.sum(torch.abs(WTW - torch.eye(WTW.shape[0])) )
loss += C
def run_app():
# dataset_textbox = st.sidebar.text_input('dataset path', value='C:\\Users\\Admin\\Downloads\\i\\n01514859\\')
DATASET_PATH = st.text_input(
'DATASET PATH', value='C:\\Users\\Admin\\Downloads\\i\\n01514859\\')
epoch_loc = st.empty()
prog_bar = st.empty()
loss_loc = st.empty()
global_loss_loc = st.empty()
loss_chart = st.empty()
glob_loss_chart = st.empty()
row0 = st.empty()
row1 = st.empty()
row2 = st.empty()
row3 = st.empty()
row4 = st.empty()
row5 = st.empty()
# st.stop()
PATH = "upscaler.pt"
net = Net()
# too lazy to detect if the file exits.
try:
net.load_state_dict(torch.load(PATH))
st.write('MODEL LOADED!')
except Exception:
pass
cuda = torch.device('cuda')
net.to(cuda)
# criterion = nn.CrossEntropyLoss()
# criterion = nn.MSELoss()
criterion = kornia.losses.PSNRLoss(1.0)
optimizer = optim.AdamW(net.parameters(), lr=0.0010)
# st.title('image upscaler')
img = load_img('image.png')
losses = deque(maxlen=100)
global_losses = deque(maxlen=100)
EPOCHS = 500
BATCH_SIZE = 1
dataset = ImageDataset(path=DATASET_PATH)
def collate_wrapper(samples):
return samples
train_loader = DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
collate_fn=collate_wrapper)
for epoch in range(EPOCHS):
i = 1
epoch_loc.write(f"EPOCH:\t{epoch}/{EPOCHS - 1}")
global_loss = torch.tensor([0.0], device=cuda)
optimizer.zero_grad()
# TODO: confirm that shuffle works
# --------------------
for batch in train_loader:
optimizer.zero_grad()
loss = torch.tensor([0.0], device=cuda)
for sample in batch:
x, y = sample
x = torch.tensor(x)
x = torch.unsqueeze(x, 0)
try:
image = x.permute(0, 3, 1, 2)
image = F.interpolate(image, size=(128, 128))
image = image.permute(0, 2, 3, 1)
row1.image(image.numpy(),
width=250,
caption='original image')
except Exception:
break
x = x.permute(0, 3, 1, 2)
y = F.interpolate(x, size=(64, 64))
x = F.interpolate(x, size=(32, 32))
x = F.interpolate(x, size=(64, 64))
row2.image(x.permute(0, 2, 3, 1).detach().numpy(),
width=250,
caption='Downsampled')
prog_bar.progress(i / len(dataset))
i += 1
out = net(x.detach().cuda().float())
diff = torch.abs(out.detach().cpu() - y.detach().cpu())
diff_image = diff.permute(0, 2, 3, 1).numpy()
row5.image(diff_image,
width=250,
caption='absolute difference')
row3.image(out.permute(0, 2, 3, 1).detach().cpu().numpy(),
width=250,
caption='Reconstructed')
loss = 1 / criterion(out, y.detach().cuda().float())
# loss = criterion(out, y.detach().cuda().float())
row4.write(f'LOSS: {loss.detach().cpu()}')
# loss.backward()
# optimizer.step()
# st.stop()
losses.append(loss.detach().cpu().numpy())
loss_chart.line_chart(pd.DataFrame(losses, columns=[
'loss',
]))
global_loss += loss
loss_loc.write(f"LOSS:\t{loss.detach().cpu()}")
loss.backward()
optimizer.step()
global_loss_loc.write(
f"GLOBAL LOSS:\t{global_loss.detach().cpu()} \nGLOB AVERAGE LOSS:\t{global_loss.detach().cpu()/len(dataset)}"
)
global_losses.append(global_loss.detach().cpu().numpy())
glob_loss_chart.line_chart(
pd.DataFrame(global_losses, columns=[
'global_loss',
]))
try:
torch.save(net.state_dict(), PATH)
st.write('MODEL SAVED!')
except Exception:
pass
def run_stage2(model, di, args, pipeline, mask_loss = False):
state = {k: v for k, v in args.items()}
if args.resume_from_best:
cp = torch.load(os.path.join(args.checkpoint,'model_best.pth.tar'))
best_acc = cp['best_acc']
best_val_loss = cp['best_loss']
else:
best_acc = 0
best_val_loss = INF
print(model)
cross_entropy_weights = torch.FloatTensor([1.0, args.pos_class_weight])
if args.cuda:
cross_entropy_weights = cross_entropy_weights.cuda()
criterion = nn.NLLLoss(weight=cross_entropy_weights)
if mask_loss:
mask_criterion = lambda x: state['mask_L1']*torch.sum(torch.abs(x))/torch.numel(x)
optimizer = optim.Adam([param for param in model.parameters() if param.requires_grad], lr=state['lr'])
start_epoch = 0
if args.evaluate:
if args.eval_on_validation:
print("Evaluating on validation set")
pipeline.evaluate_model(model, args, di, labels_avail=args.labels_avail, type='validation', mode=args.eval_mode)
else:
print("Evaluating on test set")
pipeline.evaluate_model(model, args, di, labels_avail=args.labels_avail, type='test', mode=args.eval_mode)
elif not args.epochs:
_ = pipeline.test(model, optimizer, 1, di, args, criterion)
else:
for epoch in range(start_epoch, args.epochs):
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
if mask_loss:
train_loss, train_acc = pipeline.train(model, optimizer, epoch, di, args, loss_criterion = criterion, mask_criterion = mask_criterion)
test_loss, test_acc = pipeline.test(model, optimizer, epoch, di, args, loss_criterion = criterion, mask_criterion = mask_criterion)
else:
train_loss, train_acc = pipeline.train(model, optimizer, epoch, di, args, criterion)
test_loss, test_acc = pipeline.test(model, optimizer, epoch, di, args, criterion)
# save model
# is_best = test_loss < best_val_loss
is_best = test_acc > best_acc
if is_best:
print("NEW BEST MODEL loss:{}\t auprc: {}".format(test_loss, test_acc))
best_acc = max(test_acc, best_acc)
best_val_loss = min(test_loss, best_val_loss)
pipeline.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.module.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'best_loss': best_val_loss,
'pos_class_weight': args.pos_class_weight,
'args': args,
'hold_out': args.hold_out,
'validation_list': args.validation_list,
'test_list': args.test_list,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
pipeline.adjust_learning_rate(optimizer, epoch, args, state)
def forward(self, x):
a = torch.matmul(x, self.rhs)
a_abs = torch.abs(a)
b = torch.matmul(a_abs.transpose(1, 0), self.rhs)
return b
# Transformation de diffusion de journal
scattering = Scattering1D(J, T, Q)
if use_cuda:
scattering.cuda()
x_all = x_all.cuda()
y_all = y_all.cuda()
# Calculer la transformée de diffusion pour tous les signaux du jeu de données.
Sx_all = scattering.forward(x_all)
# Supprimer les coefficients de diffusion d'ordre zéro, qui sont toujours placés dans le premier canal du tenseur de diffusion.
Sx_all = Sx_all[:,1:,:]
Sx_all = torch.log(torch.abs(Sx_all) + log_eps)
# Effectuer une moyenne sur la dernière dimension (le temps) pour obtenir une représentation invariante par décalage dans le temps.
Sx_all = torch.mean(Sx_all, dim=-1)
# Former le classificateur
# Extraire les données d'apprentissage (celles pour lesquelles le sous-ensemble = 0 ) et les étiquettes associées.
Sx_tr, y_tr = Sx_all[subset == 0], y_all[subset == 0]
# Normaliser les données pour avoir un zéro moyen et une variance d'unité.
mu_tr = Sx_tr.mean(dim=0)
std_tr = Sx_tr.std(dim=0)
Sx_tr = (Sx_tr - mu_tr) / std_tr
# Définir un modèle de régression logistique à l'aide de PyTorch, le former en utilisant Adam avec une perte de log-vraisemblance négative.
num_input = Sx_tr.shape[-1]
def build(self):
#get gradient
gradient,output_org,output_org_s,pred_org,pred_label=gen_grad(self.org_data_tensor,\
self.model, self.target_class)
print(f"output for original data (before softmax) :{torch_print(output_org)}")
print(f"output for original data :{torch_print(output_org_s)}")
print(f"prediction label for original data :{torch_print(pred_label)}")
print('-'*40+'\n')
#sorting gradient in descending order
b,c,w,h=self.org_data_tensor.shape
class_num=len(pred_org)
grad_imp_sort=grad_processing(gradient,d=self.d,w=w)
mask=np.ones((self.M,self.M))
color=np.zeros((self.M,self.M))
mask_tensor = numpy_to_torch(mask,requires_grad=False)
color_tensor = numpy_to_torch(color, requires_grad=False)
mask_num=0
color_ini=torch.rand((b,c,w,h))
#the logit scores of the training data
ref_mean,ref_var=ref_output(pred_org[self.target_class].cpu().numpy(),\
self.model,self.ref_path,class_num)
while(1):
imp_max = grad_imp_sort[0,mask_num]
imp_max_row = torch.div(imp_max, int(w / self.d))
imp_max_col = imp_max - imp_max_row * int(w / self.d)
mask_tensor[b-1,:,imp_max_row, imp_max_col] = 0
upsampled_mask = upsample(mask_tensor,w,method='near')
color_tensor[b-1,:,imp_max_row, imp_max_col] = 1
upsampled_color = upsample(color_tensor, w, method='near')
upsampled_color = torch.where(upsampled_color == 0.0, upsampled_color, color_ini.cuda())
upsampled_color = Variable(upsampled_color, requires_grad=True)
optimizer = torch.optim.Adam([upsampled_color], lr=self.lr)
for i in range(self.n_iter):
composite_tensor=self.org_data_tensor.mul(upsampled_mask)+\
upsampled_color.mul(1-upsampled_mask)
output_comp=self.model(composite_tensor) #size=[1,10]
output_comp_s=torch.nn.Softmax(dim=-1)(output_comp)
pred_label=torch.argmax(output_comp)
l2_loss= torch.dist(self.org_data_tensor, composite_tensor,2)
tv_loss= tv_norm(composite_tensor, self.tv_coeff)
# loss function
loss=torch.mean(torch.abs(output_comp-ref_mean))+ \
self.tv_coeff * tv_loss+\
self.l2_coeff*l2_loss
optimizer.zero_grad()
loss.backward(retain_graph=True)
optimizer.step()
print(f"Mask num {mask_num+1}\t Losses: total: {loss.item():3.3f},\ttv: {tv_loss.item():3.3f} \tl2: {l2_loss.item():3.3f}")
if output_comp_s[0,pred_org[self.target_class]] >= self.target_prob:
break
mask_num += 1
if mask_num==self.M*self.M:
print('Not found Counterfactual explanations')
return
slash_idx=self.data_path.rfind('/')
file_path=self.data_path[slash_idx:-4]
self.saved_path=self.saved_path+file_path+'/'
if not os.path.exists(self.saved_path):
os.makedirs(self.saved_path)
org_save_file="Org_class{}.png".format(torch_print(pred_org[0]))
per_save_file="Per_class{}.png".format(str(torch_print(pred_label)))
vutils.save_image(self.org_data_tensor.data.clone(),self.saved_path+org_save_file)
vutils.save_image(composite_tensor.data.clone(),self.saved_path+per_save_file)
print('\n'+'-' * 40)
print(f'output for composite data (before softmax) :{torch_print(output_comp)}')
print(f'output for composite data :{torch_print(output_comp_s)}')
print(f'prediction label for composite data :{torch_print(pred_label)}\n')
def model_identification(features, label, mask, poly_order, deriv_order):
lamb = 0
tolerance = 1e-6
mask = torch.ones(tot_items, 1)
print('xi', xi)
print('mask:', mask.shape)
lambd = 1e-6
L1_loss = []
MSE_loss = []
Reg_loss = []
Total_loss = []
for epoch in range(epochs):
optimizer.zero_grad()
uhat = net(features)
if epoch == 1000:
lamb = 1
dudt, theta = construct_Dictonary(features, uhat, poly_order=2, deriv_order=2)
# print('dudt:', dudt.shape)
dudt_norm = torch.norm(dudt, dim=0)
# print('dudt_norm:', dudt_norm.shape)
theta_scaling = (torch.norm(theta, dim=0))
# print('theta_scaling:', theta_scaling.shape)
# Returns a new tensor with a dimension of size one inserted at the specified position. from 9 it will be 9,1
theta_norm = torch.unsqueeze(theta_scaling, dim=1)
# print('theta_norm:', theta_norm.shape)
xi_normalized = xi * (theta_norm / dudt_norm)
L1 = lambd * torch.sum(torch.abs(xi_normalized[1:, :]))
l_u = nn.MSELoss()(uhat, label)
l_reg = lamb * torch.mean((dudt - theta @ xi) ** 2)
# l_reg = torch.mean((dudt - theta @ xi)**2)
loss = l_u + l_reg + L1
# print('loss', loss)
L1_loss.append(L1.item())
MSE_loss.append(l_u.item())
Reg_loss.append(l_reg.item())
Total_loss.append(loss.item())
losses = {"L1_loss": L1_loss,
"MSE_loss": MSE_loss,
"Reg_loss": Reg_loss,
"Total_loss": Total_loss}
gradient_loss = torch.max(torch.abs(grad(outputs=loss, inputs=xi,
grad_outputs=torch.ones_like(loss), create_graph=True)[0]) / (
theta_norm / dudt_norm))
loss.backward(retain_graph=True)
optimizer.step()
# print("epoch {}/{}, loss={:.10f}".format(epoch+1, epochs, loss.item()), end="\r")
if epoch % 1000 == 0:
print('loss:', epoch, loss)
if gradient_loss < tolerance:
print('Optimizer converged.')
break
# print('xi_normalized:', xi_normalized)
xi_list = sparse_coeff(mask, xi.detach().numpy())
xi_normalized = sparse_coeff(mask, xi_normalized.detach().numpy())
print('xi_normalized:', xi_normalized)
sparsity = normalized_xi_threshold(xi_normalized, mode='auto')
print('sparsity:', sparsity)
xi_thresholded = np.expand_dims(xi_list[sparsity], axis=1)
print('xi_thresholded:', xi_thresholded)
np.savetxt('xi1.txt', xi_thresholded[0], fmt="%.8f")
np.savetxt('xi2.txt', xi_thresholded[1], fmt="%.8f")
# Calculate Error in xi
xi1_error = np.subtract(np.array([0.10000]), xi_thresholded[0])
print('xi1_error', xi1_error)
xi2_error = np.subtract(np.array([1.00000]), np.abs(xi_thresholded[1]))
print('xi2_error', xi2_error)
print('Coefficient xi:')
xi_updated = sparse_coeff(sparsity, xi_thresholded)
print(xi_updated)
print('Finished')
return xi_updated, losses
def cubic(x):
absx = torch.abs(x)
absx2 = absx**2
absx3 = absx**3
return (1.5*absx3 - 2.5*absx2 + 1) * ((absx <= 1).type_as(absx)) + \
(-0.5*absx3 + 2.5*absx2 - 4*absx + 2) * (((absx > 1)*(absx <= 2)).type_as(absx))
def navier_stokes_2d_(w0, f, visc, T, delta_t=1e-4, record_steps=1):
#Grid size - must be power of 2
N = w0.size()[-1]
#Maximum frequency
k_max = math.floor(N / 2.0)
#Number of steps to final time
steps = math.ceil(T / delta_t)
#Initial vorticity to Fourier space
w_h = torch.rfft(w0, 2, normalized=False, onesided=False)
#Forcing to Fourier space
f_h = torch.rfft(f, 2, normalized=False, onesided=False)
#If same forcing for the whole batch
if len(f_h.size()) < len(w_h.size()):
f_h = torch.unsqueeze(f_h, 0)
#Record solution every this number of steps
record_time = math.floor(steps / record_steps)
#Wavenumbers in y-direction
k_y = torch.cat(
(torch.arange(start=0, end=k_max, step=1, device=w0.device),
torch.arange(start=-k_max, end=0, step=1, device=w0.device)),
0).repeat(N, 1)
#Wavenumbers in x-direction
k_x = k_y.transpose(0, 1)
#Negative Laplacian in Fourier space
lap = 4 * (math.pi**2) * (k_x**2 + k_y**2)
lap[0, 0] = 1.0
#Dealiasing mask
dealias = torch.unsqueeze(
torch.logical_and(
torch.abs(k_y) <= (2.0 / 3.0) * k_max,
torch.abs(k_x) <= (2.0 / 3.0) * k_max).float(), 0)
#Saving solution and time
sol = torch.zeros(*w0.size(), record_steps, device=w0.device)
sol_t = torch.zeros(record_steps, device=w0.device)
#Record counter
c = 0
#Physical time
t = 0.0
for j in range(steps):
#Stream function in Fourier space: solve Poisson equation
psi_h = w_h.clone()
psi_h[..., 0] = psi_h[..., 0] / lap
psi_h[..., 1] = psi_h[..., 1] / lap
#Velocity field in x-direction = psi_y
q = psi_h.clone()
temp = q[..., 0].clone()
q[..., 0] = -2 * math.pi * k_y * q[..., 1]
q[..., 1] = 2 * math.pi * k_y * temp
q = torch.irfft(q,
2,
normalized=False,
onesided=False,
signal_sizes=(N, N))
#Velocity field in y-direction = -psi_x
v = psi_h.clone()
temp = v[..., 0].clone()
v[..., 0] = 2 * math.pi * k_x * v[..., 1]
v[..., 1] = -2 * math.pi * k_x * temp
v = torch.irfft(v,
2,
normalized=False,
onesided=False,
signal_sizes=(N, N))
#Partial x of vorticity
w_x = w_h.clone()
temp = w_x[..., 0].clone()
w_x[..., 0] = -2 * math.pi * k_x * w_x[..., 1]
w_x[..., 1] = 2 * math.pi * k_x * temp
w_x = torch.irfft(w_x,
2,
normalized=False,
onesided=False,
signal_sizes=(N, N))
#Partial y of vorticity
w_y = w_h.clone()
temp = w_y[..., 0].clone()
w_y[..., 0] = -2 * math.pi * k_y * w_y[..., 1]
w_y[..., 1] = 2 * math.pi * k_y * temp
w_y = torch.irfft(w_y,
2,
normalized=False,
onesided=False,
signal_sizes=(N, N))
#Non-linear term (u.grad(w)): compute in physical space then back to Fourier space
F_h = torch.rfft(q * w_x + v * w_y,
2,
normalized=False,
onesided=False)
#Dealias
F_h[..., 0] = dealias * F_h[..., 0]
F_h[..., 1] = dealias * F_h[..., 1]
#Cranck-Nicholson update
w_h[..., 0] = (-delta_t * F_h[..., 0] + delta_t * f_h[..., 0] +
(1.0 - 0.5 * delta_t * visc * lap) * w_h[..., 0]) / (
1.0 + 0.5 * delta_t * visc * lap)
w_h[..., 1] = (-delta_t * F_h[..., 1] + delta_t * f_h[..., 1] +
(1.0 - 0.5 * delta_t * visc * lap) * w_h[..., 1]) / (
1.0 + 0.5 * delta_t * visc * lap)
#Update real time (used only for recording)
t += delta_t
if (j + 1) % record_time == 0:
#Solution in physical space
w = torch.irfft(w_h,
2,
normalized=False,
onesided=False,
signal_sizes=(N, N))
#Record solution and time
sol[..., c] = w
sol_t[c] = t
c += 1
return sol # , sol_t
module_name='PReLU',
input_size=(2, 3, 4, 5, 6),
reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
desc='3d',
),
dict(
module_name='PReLU',
constructor_args=(3,),
input_size=(2, 3, 4, 5, 6),
desc='3d_multiparam',
reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
),
dict(
module_name='Softsign',
input_size=(3, 2, 5),
reference_fn=lambda i, _: i.div(1 + torch.abs(i)),
),
dict(
module_name='Softmin',
constructor_args=(1,),
input_size=(10, 20),
),
dict(
module_name='Softmin',
constructor_args=(1,),
input_size=(2, 3, 5, 10),
desc='multidim',
),
dict(
module_name='Tanhshrink',
input_size=(2, 3, 4, 5)
def DAGH_algo(code_length, dataname):
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu
torch.manual_seed(0)
torch.cuda.manual_seed(0)
# code_length=8
'''
parameter setting
'''
max_iter = opt.max_iter
epochs = opt.epochs
batch_size = opt.batch_size
learning_rate = opt.learning_rate
weight_decay = 5 * 10**-4
num_anchor = opt.num_anchor
lambda_1 = float(opt.lambda_1)
lambda_2 = float(opt.lambda_2)
lambda_3 = float(opt.lambda_3)
record['param']['opt'] = opt
record['param']['description'] = '[Comment: learning rate decay]'
logger.info(opt)
logger.info(code_length)
logger.info(record['param']['description'])
'''
dataset preprocessing
'''
nums, dsets, labels = _dataset(dataname)
num_database, num_test = nums
dset_database, dset_test = dsets
database_labels, test_labels = labels
'''
model construction
'''
beta = 2
model = cnn_model.CNNNet(opt.arch, code_length)
model.cuda()
cudnn.benchmark = True
DAGH_loss = dl.DAGHLoss(lambda_1, lambda_2, lambda_3, code_length)
L1_criterion = nn.L1Loss()
L2_criterion = nn.MSELoss()
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
B = np.sign(np.random.randn(code_length, num_database))
model.train()
for iter in range(max_iter):
iter_time = time.time()
trainloader = DataLoader(dset_database,
batch_size=batch_size,
shuffle=True,
num_workers=4)
F = np.zeros((num_database, code_length), dtype=np.float)
if iter == 0:
'''
initialize the feature of all images to build dist graph
'''
ini_Features = np.zeros((num_database, 4096), dtype=np.float)
ini_F = np.zeros((num_database, code_length), dtype=np.float)
for iteration, (train_input, train_label,
batch_ind) in enumerate(trainloader):
train_input = Variable(train_input.cuda())
output = model(train_input)
ini_Features[batch_ind, :] = output[0].cpu().data.numpy()
ini_F[batch_ind, :] = output[1].cpu().data.numpy()
print('initialization dist graph forward done!')
dist_graph = get_dist_graph(ini_Features, num_anchor)
# dist_graph = np.random.rand(num_database,num_anchor)
# bf = np.sign(ini_F)
Z = calc_Z(dist_graph)
elif (iter % 3) == 0:
dist_graph = get_dist_graph(Features, num_anchor)
Z = calc_Z(dist_graph)
print('calculate dist graph forward done!')
inv_A = inv(np.diag(Z.sum(0))) # m X m
Z_T = Z.transpose() # m X n
left = np.dot(B, np.dot(Z, inv_A)) # k X m
if iter == 0:
loss_ini = calc_loss(B, ini_F, Z, inv_A, lambda_1, lambda_2,
lambda_3, code_length)
# loss_ini2 = calc_all_loss(B,F,Z,inv_A,Z1,Z2,Y1,Y2,rho1,rho2,lambda_1,lambda_2)
print(loss_ini)
'''
learning deep neural network: feature learning
'''
Features = np.zeros((num_database, 4096), dtype=np.float)
for epoch in range(epochs):
for iteration, (train_input, train_label,
batch_ind) in enumerate(trainloader):
train_input = Variable(train_input.cuda())
output = model(train_input)
Features[batch_ind, :] = output[0].cpu().data.numpy()
F[batch_ind, :] = output[1].cpu().data.numpy()
batch_grad = get_batch_gard(
B, left, Z_T, batch_ind) / (code_length * batch_size)
batch_grad = Variable(
torch.from_numpy(batch_grad).type(
torch.FloatTensor).cuda())
optimizer.zero_grad()
# output[1].backward(batch_grad, retain_graph=True)
output[1].backward(batch_grad)
B_cuda = Variable(
torch.from_numpy(B[:, batch_ind]).type(
torch.FloatTensor).cuda())
# optimizer.zero_grad ()
other_loss = DAGH_loss(output[1].t(), B_cuda)
one_vectors = Variable(torch.ones(output[1].size()).cuda())
L1_loss = L1_criterion(torch.abs(output[1]), one_vectors)
# L2_loss = L2_criterion (output[1],B_cuda.t())
All_loss = other_loss + lambda_3 * L1_loss / code_length
All_loss.backward()
optimizer.step()
if (iteration % 200) == 0:
print('iteration:' + str(iteration))
#print (model.features[0].weight.data[1, 1, :, :])
#print (model.features[18].weight.data[1, 1, :, :])
#print (model.classifier[6].weight.data[:, 1])
adjusting_learning_rate(optimizer, iter)
'''
learning binary codes: discrete coding
'''
# bf = np.sign (F)
# F = np.random.randn (num_database, 12)
loss_before = calc_loss(B, F, Z, inv_A, lambda_1, lambda_2, lambda_3,
code_length)
B = B_step(F, Z, inv_A)
iter_time = time.time() - iter_time
loss_ = calc_loss(B, F, Z, inv_A, lambda_1, lambda_2, lambda_3,
code_length)
logger.info(
'[Iteration: %3d/%3d][Train Loss: before:%.4f, after:%.4f]', iter,
max_iter, loss_before, loss_)
record['train loss'].append(loss_)
record['iter time'].append(iter_time)
'''
training procedure finishes, evaluation
'''
model.eval()
testloader = DataLoader(dset_test,
batch_size=batch_size,
shuffle=False,
num_workers=4)
qB = encode(model, testloader, num_test, code_length)
rB = B.transpose()
topKs = np.arange(1, 500, 50)
top_ndcg = 100
map = calc_hr.calc_map(qB, rB, test_labels.numpy(),
database_labels.numpy())
top_map = calc_hr.calc_topMap(qB, rB, test_labels.numpy(),
database_labels.numpy(), 2000)
Pres = calc_hr.calc_topk_pres(qB, rB, test_labels.numpy(),
database_labels.numpy(), topKs)
ndcg = calc_hr.cal_ndcg_k(qB, rB, test_labels.numpy(),
database_labels.numpy(), top_ndcg)
logger.info('[lambda_1: %.4f]', lambda_1)
logger.info('[lambda_2: %.4f]', lambda_2)
logger.info('[lambda_3: %.4f]', lambda_3)
logger.info('[Evaluation: mAP: %.4f]', map)
logger.info('[Evaluation: topK_mAP: %.4f]', top_map)
logger.info('[Evaluation: Pres: %.4f]', Pres[0])
logger.info('[Evaluation: topK_ndcg: %.4f]', ndcg)
record['rB'] = rB
record['qB'] = qB
record['map'] = map
record['topK_map'] = top_map
record['topK_ndcg'] = ndcg
record['Pres'] = Pres
record['F'] = F
filename = os.path.join(logdir, str(code_length) + 'bits-record.pkl')
_save_record(record, filename)
return top_map
def median_absolute_percentage_error_compute_fn(y_pred: torch.Tensor, y: torch.Tensor):
e = torch.abs(y.view_as(y_pred) - y_pred) / torch.abs(y.view_as(y_pred))
return 100.0 * torch.median(e).item()
def forward(self, x1, x2):
y1 = self.forward_one(x1)
y2 = self.forward_one(x2)
diff = torch.abs(y1 - y2)
out = self.out(diff)
return out
G.load_state_dict(torch.load('./WGAN_for_DOS/generator_MP2020.pt'))
G.eval()
def sample_generator(G, num_samples, feature):
generated_data_all = 0
num_sam = 100
for i in range(num_sam):
latent_samples = Variable(G.sample_latent(num_samples))
latent_samples = latent_samples
generated_data = G(torch.cat((feature, latent_samples), dim=1))
generated_data_all += generated_data
generated_data = generated_data_all/num_sam
return generated_data
print('Testing...')
gen_data = sample_generator(G, batch_size, test_feat).detach()
print('Done!')
#gen_data = gen_data*std+mean
test_label = (test_label-mean)/std
MAE = torch.mean(torch.abs(gen_data-test_label))
print('MAE:', MAE.numpy())
# save testing results
np.save('./WGAN_for_DOS/test_label.npy', test_label)
np.save('./WGAN_for_DOS/test_pred.npy', gen_data)
def compute_depth_metrics(config, gt, pred, use_gt_scale=True):
"""
Compute depth metrics from predicted and ground-truth depth maps
Parameters
----------
config : CfgNode
Metrics parameters
gt : torch.Tensor [B,1,H,W]
Ground-truth depth map
pred : torch.Tensor [B,1,H,W]
Predicted depth map
use_gt_scale : bool
True if ground-truth median-scaling is to be used
Returns
-------
metrics : torch.Tensor [7]
Depth metrics (abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3)
"""
crop = config.crop == 'garg'
# Initialize variables
batch_size, _, gt_height, gt_width = gt.shape
abs_diff = abs_rel = sq_rel = rmse = rmse_log = a1 = a2 = a3 = 0.0
# Interpolate predicted depth to ground-truth resolution
pred = interpolate_image(pred,
gt.shape,
mode='bilinear',
align_corners=True)
# If using crop
if crop:
crop_mask = torch.zeros(gt.shape[-2:]).byte().type_as(gt)
y1, y2 = int(0.40810811 * gt_height), int(0.99189189 * gt_height)
x1, x2 = int(0.03594771 * gt_width), int(0.96405229 * gt_width)
crop_mask[y1:y2, x1:x2] = 1
# For each depth map
for pred_i, gt_i in zip(pred, gt):
gt_i, pred_i = torch.squeeze(gt_i), torch.squeeze(pred_i)
# Keep valid pixels (min/max depth and crop)
valid = (gt_i > config.min_depth) & (gt_i < config.max_depth)
valid = valid & crop_mask.bool() if crop else valid
# Stop if there are no remaining valid pixels
if valid.sum() == 0:
continue
# Keep only valid pixels
gt_i, pred_i = gt_i[valid], pred_i[valid]
# Ground-truth median scaling if needed
if use_gt_scale:
pred_i = pred_i * torch.median(gt_i) / torch.median(pred_i)
# Clamp predicted depth values to min/max values
pred_i = pred_i.clamp(config.min_depth, config.max_depth)
# Calculate depth metrics
thresh = torch.max((gt_i / pred_i), (pred_i / gt_i))
a1 += (thresh < 1.25).float().mean()
a2 += (thresh < 1.25**2).float().mean()
a3 += (thresh < 1.25**3).float().mean()
diff_i = gt_i - pred_i
abs_diff += torch.mean(torch.abs(diff_i))
abs_rel += torch.mean(torch.abs(diff_i) / gt_i)
sq_rel += torch.mean(diff_i**2 / gt_i)
rmse += torch.sqrt(torch.mean(diff_i**2))
rmse_log += torch.sqrt(
torch.mean((torch.log(gt_i) - torch.log(pred_i))**2))
# Return average values for each metric
return torch.tensor([
metric / batch_size
for metric in [abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3]
]).type_as(gt)
def _forward(self, pred, target, weight):
return torch.mean(weight * torch.abs(pred - target))
def forward(self, beta, bias):
bias_ = bias.view(1, -1)
tot = torch.cat((beta, bias_), dim=0)
return torch.sum(torch.abs(tot))
loader: the dataloader of test or validation set
device: either CPU or CUDA
"""
model.eval()
accuracies = []
losses = []
with torch.no_grad:
for idx, thebatch in enumerate(trainloader):
batch = torch.Tensor(batch["image"]).to(device)
depth = torch.Tensor(batch["depth"]).to(device=device)
"""
for batch, truth in loader:
batch = batch.to(device)
truth = truth.to(device)"""
pred = model(batch)
accuracies.append(torch.mean(torch.abs(pred - truth) / truth).item())
loss = DepthLoss(0.1)
losses.append(loss(pred, truth).item())
acc = sum(accuracies) / len(accuracies)
loss = sum(losses) / len(losses)
print("Evaluation accuracy: {}".format(acc))
return acc, loss
def train_epoch(data_loader, model, criterion, optimizer):
"""
Train the `model` for one epoch of data from `data_loader`.
Use `optimizer` to optimize the specified `criterion`
"""
# for i, (X, y) in enumerate(data_loader):
def forward(self, x):
norm = torch.sum(torch.abs(x), dim = 1) + self.eps
x= x / norm.expand_as(x)
return x
def lp_norm(x, alpha):
""" calculate LP-norm loss """
return torch.abs(x.view(-1) ** alpha).sum() / np.prod(x.shape)
def classify(self, feature1, feature2): #check for similarity here
x = torch.abs(feature2 - feature1)
x = self.linear2(x)
return torch.sigmoid(x)
def tv_norm(input, tv_beta):
img = input[0, 0, :]
row_grad = torch.mean(torch.abs((img[:-1, :] -img[1:, :])).pow(tv_beta))
col_grad = torch.mean(torch.abs((img[:, :-1] - img[:, 1:])).pow(tv_beta))
return (row_grad + col_grad)
def _compute_loss_smooth(self, mat):
return torch.sum(torch.abs(mat[:, :, :, :-1] - mat[:, :, :, 1:])) + \
torch.sum(torch.abs(mat[:, :, :-1, :] - mat[:, :, 1:, :]))
starttime = time.time()
qc = RunningQuantile(resolution=6 * 1024)
qc.add(alldata)
# Test state dict
saved = qc.state_dict()
# numpy.savez('foo.npz', **saved)
# saved = numpy.load('foo.npz')
qc = RunningQuantile(state=saved)
assert not qc.device.type == 'cuda'
qc.add(alldata)
actual_sum *= 2
ro = qc.readout(1001).cpu()
endtime = time.time()
gt = torch.linspace(0, amount, quantiles + 1)[None, :] + (
torch.arange(qc.depth, dtype=torch.float) * amount)[:, None]
maxreldev = torch.max(torch.abs(ro - gt) / amount) * quantiles
print("Maximum relative deviation among %d perentiles: %f" %
(quantiles, maxreldev))
minerr = torch.max(
torch.abs(qc.minmax().cpu()[:, 0] -
torch.arange(qc.depth, dtype=torch.float) * amount))
maxerr = torch.max(
torch.abs((qc.minmax().cpu()[:, -1] + 1) -
(torch.arange(qc.depth, dtype=torch.float) + 1) * amount))
print("Minmax error %f, %f" % (minerr, maxerr))
interr = torch.max(
torch.abs(qc.integrate(lambda x: x * x).cpu() - actual_sum) /
actual_sum)
print("Integral error: %f" % interr)
medianerr = torch.max(
torch.abs(qc.median() - alldata.median(0)[0]) /
print("数字%s对应的图片是:"%(image_label[1]))
cv2.imshow(str(image_label[1]),np.array(image_data[1]).reshape((28,28)).astype('uint8'))
cv2.waitKey(2000)
print("-"*20)
'''
lr = float(sys.argv[1])
# 对模型进行训练：
weights0,weight1=train_model(image_data,image_label,weights0,weights1,lr)
#测试：
correct=0
for i in range(80,100):
#print(image_label[i])
#y = model(get_feature(image_data[i]),weights)
feature = get_feature(image_data[i])
y = model(feature,weights0,weights1)
#pdb.set_trace()
gt = image_label[i]
#pred=torch.argmin(torch.abs(y-gt)).item()
#pred = torch.argmin(torch.from_numpy(np.array([torch.min((torch.abs(y-j))).item() for j in range(0,10)]))).item()
pred = torch.argmin(torch.min(torch.abs(y-1))).item()
print("图像[%s]得分类结果是:[%s]"%(gt,pred))
if gt==pred:
correct+=1
print("acc=%s"%(float(correct/20.0)))
def test(self):
self.restore_model(self.resume_iters)
self.G.eval()
self.device = torch.device('cpu')
self.G.to(self.device)
self.iteration = self.test_len
self.graph_size = self.test_size
self.model_save_dir = os.path.join(self.checkpoint_dir, self.model_dir)
self.Th_error = np.load(self.model_save_dir + '_threshold.npy')
tp = 0.
tn = 0.
fp = 0.
fn = 0.
with torch.no_grad():
for idx in range(self.iteration):
# =================================================================================== #
# 1. Preprocess input data #
# =================================================================================== #
node_path = self.dataset_name + '/node/testnode' + str(
idx + 1) + '.npy'
edge_path = self.dataset_name + '/graph/testgraph' + str(
idx + 1) + '.npy'
ab_path = self.dataset_name + '/abnormal/abnormal' + str(
idx + 1) + '.npy'
node_feature, edge, _, abnormal = load_graph(
node_path, edge_path, abnormal_path=ab_path)
help = torch.eye(edge.shape[1], dtype=torch.float)
node_exist = torch.sum(torch.mul(
help, edge), dim=-1) # whether or not the node exists
edge = torch.abs(torch.mul(1. - help, edge).to(self.device))
node_feature = node_feature.to(self.device)
# =================================================================================== #
# 2. Train the Auto-encoder #
# =================================================================================== #
recon_a, recon_x, node_embedding = self.G(node_feature, edge)
a_score = self.loss_function(recon_a, edge, graph=False)
x_score = self.loss_function(recon_x,
node_feature,
graph=False)
Anomaly_score = (self.ax_w * a_score +
(1 - self.ax_w) * x_score)
record1 = (Anomaly_score > self.Th_error[0]).float().cpu()
for n in range(record1.shape[0]):
if record1[n] == abnormal[n]:
if record1[n] == 0:
tp += 1.
else:
tn += 1.
else:
if record1[n] == 0:
fp += 1.
else:
fn += 1.
torch.cuda.empty_cache()
acc = (tp + tn) / (tp + tn + fp + fn)
recall = tp / (tp + fn)
prec = tp / (tp + fp)
f1 = 2 * (recall * prec) / (recall + prec)
tnr = tn / (tn + fp)
print(acc)
print(recall)
print(prec)
print(f1)
print(tnr)
def train_epoch(device,
logger,
epoch,
trainer,
train_ds,
val_ds,
train_batch_size,
val_batch_size,
num_workers,
save_every,
eval_every,
save_imgs_every,
train_eval_indices,
val_eval_indices,
tb_log_every=100,
tb_log_enc_every=500,
n_au_steps=1,
dbg=False):
""""""
# log buffers
au_loss_buffer = []
au_loss_on_real_buffer = []
au_loss_on_fake_buffer = []
au_reg_buffer = []
au_out_on_real_buffer = []
au_out_on_fake_buffer = []
au_pred_on_real_buffer = []
au_pred_on_fake_buffer = []
im_loss_buffer = []
trainloader = DataLoader(train_ds,
batch_size=train_batch_size,
shuffle=True,
num_workers=num_workers,
drop_last=True)
num_iters = 50 if dbg else len(trainloader)
iter_bar = tqdm(itertools.islice(trainloader, num_iters),
total=num_iters,
desc='Training')
for batch_idx, data_batch in enumerate(iter_bar):
# step
trainer.module.do_global_step()
trainer.module.update_learning_rate()
# data
real_sample = data_batch["real_sample"].to(device)
leaked_sample = data_batch["leaked_sample"].to(device)
si_sample = data_batch["si_sample"].to(device)
global_step = trainer.module.global_step
# impersonator train step
if (global_step + 1) % n_au_steps == 0:
im_loss, fake_sample, _ = im_train_step(
trainer=trainer,
leaked_sample=leaked_sample,
si_sample=si_sample)
else:
im_loss, fake_sample, _ = im_eval_step(trainer=trainer,
leaked_sample=leaked_sample,
si_sample=si_sample)
im_loss_buffer.append(im_loss.view(1))
# authenticator train step
au_loss, au_loss_on_real, au_loss_on_fake, au_reg, au_out_on_real, au_out_on_fake, au_pred_on_real, au_pred_on_fake, fake_sample = au_train_step(
trainer=trainer,
real_sample=real_sample,
fake_sample=fake_sample,
si_sample=si_sample)
# log stats
au_loss_buffer.append(au_loss.view(1))
au_loss_on_real_buffer.append(au_loss_on_real.view(1))
au_loss_on_fake_buffer.append(au_loss_on_fake.view(1))
au_reg_buffer.append(au_reg.view(1))
au_out_on_real_buffer.append(au_out_on_real.view(1))
au_out_on_fake_buffer.append(au_out_on_fake.view(1))
au_pred_on_real_buffer.append(au_pred_on_real.view(-1))
au_pred_on_fake_buffer.append(au_pred_on_fake.view(-1))
if global_step % tb_log_every == 0:
# lr
logger.add_scalar(category='lr',
k='au',
v=trainer.module.au_lr,
global_step=global_step)
logger.add_scalar(category='lr',
k='im',
v=trainer.module.im_lr,
global_step=global_step)
logger.add_scalar(category='lr',
k='im_lm',
v=trainer.module.im_noise_mapping_lr,
global_step=global_step)
# losses
logger.add_scalar(category='train_losses',
k='dis_loss',
v=torch.cat(au_loss_buffer).mean().item(),
global_step=global_step)
logger.add_scalar(
category='train_losses',
k='dis_loss_on_real',
v=torch.cat(au_loss_on_real_buffer).mean().item(),
global_step=global_step)
logger.add_scalar(
category='train_losses',
k='dis_loss_on_fake',
v=torch.cat(au_loss_on_fake_buffer).mean().item(),
global_step=global_step)
logger.add_scalar(category='train_losses',
k='dis_reg',
v=torch.cat(au_reg_buffer).mean().item(),
global_step=global_step)
logger.add_scalar(category='train_au_out',
k='au_out_on_real',
v=torch.cat(au_out_on_real_buffer).mean().item(),
global_step=global_step)
logger.add_scalar(category='train_au_out',
k='au_out_on_fake',
v=torch.cat(au_out_on_fake_buffer).mean().item(),
global_step=global_step)
# acc
au_acc_on_real = torch.cat(au_pred_on_real_buffer).to(
torch.float).mean()
au_acc_on_fake = torch.eq(torch.cat(au_pred_on_fake_buffer),
0).to(torch.float).mean()
au_acc = 0.5 * (au_acc_on_real + au_acc_on_fake)
logger.add_scalar(category='train_accuracy',
k='dis_acc',
v=au_acc.item(),
global_step=global_step)
logger.add_scalar(category='train_accuracy',
k='dis_acc_on_real',
v=au_acc_on_real.item(),
global_step=global_step)
logger.add_scalar(category='train_accuracy',
k='dis_acc_on_fake',
v=au_acc_on_fake.item(),
global_step=global_step)
# im
if len(im_loss_buffer) > 0:
logger.add_scalar(category='train losses',
k='gen loss',
v=torch.cat(im_loss_buffer).mean().item(),
global_step=global_step)
# clear buffers
au_loss_buffer = []
au_loss_on_real_buffer = []
au_loss_on_fake_buffer = []
au_reg_buffer = []
au_out_on_real_buffer = []
au_out_on_fake_buffer = []
au_pred_on_real_buffer = []
au_pred_on_fake_buffer = []
im_loss_buffer = []
# log encodings
if global_step % tb_log_enc_every == 0:
with torch.no_grad():
# authenticator
au_real_src = trainer.module.authenticator.src_encode_sample(
real_sample)
au_si_src = trainer.module.authenticator.src_encode_sample(
si_sample)
au_fake_src = trainer.module.authenticator.src_encode_sample(
fake_sample)
au_real_env = trainer.module.authenticator.env_encode_sample(
real_sample)
au_si_env = trainer.module.authenticator.env_encode_sample(
si_sample)
au_fake_env = trainer.module.authenticator.env_encode_sample(
fake_sample)
# mean
logger.add_scalar(
category='train-au_src_mean',
k='abs[real-si]',
v=torch.abs(au_real_src.mean(1) -
au_si_src.mean(1)).mean().item(),
global_step=global_step)
logger.add_scalar(
category='train-au_src_mean',
k='abs[fake-si]',
v=torch.abs(au_fake_src.mean(1) -
au_si_src.mean(1)).mean().item(),
global_step=global_step)
logger.add_scalar(
category='train-au_env_mean',
k='abs[real-si]',
v=torch.abs(au_real_env.mean(1) -
au_si_env.mean(1)).mean().item(),
global_step=global_step)
logger.add_scalar(
category='train-au_env_mean',
k='abs[fake-si]',
v=torch.abs(au_fake_env.mean(1) -
au_si_env.mean(1)).mean().item(),
global_step=global_step)
# std
au_real_src_std = mb.custom_std(au_real_src).mean().item()
au_si_src_std = mb.custom_std(au_si_src).mean().item()
au_fake_src_std = mb.custom_std(au_fake_src).mean().item()
logger.add_scalar(category='train-au_src_std',
k='real',
v=au_real_src_std,
global_step=global_step)
logger.add_scalar(category='train-au_src_std',
k='si',
v=au_si_src_std,
global_step=global_step)
logger.add_scalar(category='train-au_src_std',
k='fake',
v=au_fake_src_std,
global_step=global_step)
au_real_env_std = mb.custom_std(au_real_env).mean().item()
au_si_env_std = mb.custom_std(au_si_env).mean().item()
au_fake_env_std = mb.custom_std(au_fake_env).mean().item()
logger.add_scalar(category='train-au_env_std',
k='real',
v=au_real_env_std,
global_step=global_step)
logger.add_scalar(category='train-au_env_std',
k='si',
v=au_si_env_std,
global_step=global_step)
logger.add_scalar(category='train-au_env_std',
k='fake',
v=au_fake_env_std,
global_step=global_step)
if (global_step % save_every == 0):
trainer.module.save(epoch=epoch)
if global_step % save_imgs_every == 0:
sample_and_save_imgs(device=device,
logger=logger,
trainer=trainer,
ds=train_ds,
ds_prefix='train',
indices=train_eval_indices,
dbg=dbg)
sample_and_save_imgs(device=device,
logger=logger,
trainer=trainer,
ds=val_ds,
ds_prefix='val',
indices=val_eval_indices,
dbg=dbg)
if global_step % eval_every == 0:
eval_step(device=device,
trainer=trainer,
ds=val_ds,
logger=logger,
batch_size=val_batch_size)
def _run_one_fw(self, pixel_model, pixel_inp, cat_var, target, base_eps, avoid_target=True):
batch_size, channels, height, width = pixel_inp.size()
pixel_inp_jpeg = self._jpeg_cat(pixel_inp, cat_var, base_eps, batch_size, height, width)
s = pixel_model(pixel_inp_jpeg)
for it in range(self.nb_its):
loss = self.criterion(s, target)
loss.backward()
if avoid_target:
grad = cat_var.grad.data
else:
grad = -cat_var.grad.data
def where_float(cond, if_true, if_false):
return cond.float() * if_true + (1-cond.float()) * if_false
def where_long(cond, if_true, if_false):
return cond.long() * if_true + (1-cond.long()) * if_false
abs_grad = torch.abs(grad).view(batch_size, -1)
num_pixels = abs_grad.size()[1]
sign_grad = torch.sign(grad)
bound = where_float(sign_grad > 0, self.l1_max - cat_var, cat_var + self.l1_max).view(batch_size, -1)
k_min = torch.zeros((batch_size,1), dtype=torch.long, requires_grad=False, device='cuda')
k_max = torch.ones((batch_size,1), dtype=torch.long, requires_grad=False, device='cuda') * num_pixels
# cum_bnd[k] is meant to track the L1 norm we end up with if we take
# the k indices with the largest gradient magnitude and push them to their boundary values (0 or 255)
values, indices = torch.sort(abs_grad, descending=True)
bnd = torch.gather(bound, 1, indices)
# subtract bnd because we don't want the cumsum to include the final element
cum_bnd = torch.cumsum(bnd, 1) - bnd
# this is hard-coded as floor(log_2(256 * 256 * 3))
for _ in range(17):
k_mid = (k_min + k_max) // 2
l1norms = torch.gather(cum_bnd, 1, k_mid)
k_min = where_long(l1norms > base_eps, k_min, k_mid)
k_max = where_long(l1norms > base_eps, k_mid, k_max)
# next want to set the gradient of indices[0:k_min] to their corresponding bound
magnitudes = torch.zeros((batch_size, num_pixels), requires_grad=False, device='cuda')
for bi in range(batch_size):
magnitudes[bi, indices[bi, :k_min[bi,0]]] = bnd[bi, :k_min[bi,0]]
magnitudes[bi, indices[bi, k_min[bi,0]]] = base_eps[bi] - cum_bnd[bi, k_min[bi,0]]
delta_it = sign_grad * magnitudes.view(cat_var.size())
# These should always be exactly epsilon
# l1_check = torch.norm(delta_it.view(batch_size, -1), 1.0, dim=1) / num_pixels
# print('l1_check: %s' % l1_check)
cat_var.data = cat_var.data + (delta_it - cat_var.data) / (it + 1.0)
if it != self.nb_its - 1:
# self.jpeg scales rounding_vars by base_eps, so we divide to rescale
# its coordinates to [-1, 1]
cat_var_temp = cat_var / base_eps[:, None]
pixel_inp_jpeg = self._jpeg_cat(pixel_inp, cat_var_temp, base_eps, batch_size, height, width)
s = pixel_model(pixel_inp_jpeg)
cat_var.grad.data.zero_()
return cat_var
def Sharpen(x, v):
return sharpen(x, torch.abs(v))
def NJ_loss(ypred):
'''
Penalizing locations where Jacobian has negative determinants
'''
Neg_Jac = 0.5 * (torch.abs(Get_Ja(ypred)) - Get_Ja(ypred))
return torch.sum(Neg_Jac)
def train(self):
start_iters = self.resume_iters if not self.new_start else 0
self.restore_model(self.resume_iters)
self.iteration = self.train_len
self.graph_size = self.train_size
start_epoch = (int)(start_iters / self.iteration)
start_batch_id = start_iters - start_epoch * self.iteration
# loop for epoch
start_time = time.time()
lr = self.init_lr
self.set_requires_grad([self.G], True)
self.G.train()
for epoch in range(start_epoch, self.epoch):
if self.decay_flag and epoch > self.decay_epoch:
lr = self.init_lr * (self.epoch - epoch) / (
self.epoch - self.decay_epoch) # linear decay
self.update_lr(lr)
for idx in range(start_batch_id, self.iteration):
# =================================================================================== #
# 1. Preprocess input data #
# =================================================================================== #
node_path = self.dataset_name + '/node/node' + str(idx +
1) + '.npy'
edge_path = self.dataset_name + '/graph/graph' + str(
idx + 1) + '.npy'
node_feature, edge, _, _ = load_graph(node_path, edge_path)
node_feature = node_feature.float()
edge = edge.float()
help = torch.eye(edge.shape[1], dtype=torch.float)
node_exist = torch.sum(torch.mul(
help, edge), dim=-1) # whether or not the node exists
edge = torch.abs(torch.mul(1. - help, edge)).to(self.device)
node_feature = node_feature.to(self.device)
loss = {}
# =================================================================================== #
# 2. Train the Auto-encoder #
# =================================================================================== #
recon_a, recon_x, _ = self.G(node_feature, edge)
self.recon_a_error = self.loss_function(recon_a, edge)
self.recon_x_error = self.loss_function(recon_x, node_feature)
self.Reconstruction_error = (
self.ax_w * self.recon_a_error +
(1 - self.ax_w) * self.recon_x_error)
# Logging.
loss['Edge_reconstruction_error'] = self.recon_a_error.item()
loss['feature_reconstruction_error'] = self.recon_x_error.item(
)
# loss['G/loss_cycle'] = self.cycle_loss.item()
loss['Reconstruction_error'] = self.Reconstruction_error.item()
del recon_a
del recon_x
torch.cuda.empty_cache()
self.reset_grad()
self.Reconstruction_error.backward()
self.g_optimizer.step()
# =================================================================================== #
# 4. Miscellaneous #
# =================================================================================== #
start_iters += 1
# Print out training information.
if idx % self.print_freq == 0:
et = time.time() - start_time
et = str(datetime.timedelta(seconds=et))[:-7]
log = "Elapsed [{}], Epoch [{}/{}], Iteration [{}/{}]".format(
et, epoch + 1, self.epoch, idx + 1, self.iteration)
for tag, value in loss.items():
if 'error' in tag: # != 'G/lable' and tag !='O/lable':
log += ", {}: {:.4f}".format(tag, value)
if self.use_tensorboard:
self.logger.scalar_summary(
tag, value, start_iters)
print(log)
torch.cuda.empty_cache()
# Save model checkpoints.
if (idx + 1) % self.save_freq == 0:
self.save(self.checkpoint_dir, start_iters)
torch.cuda.empty_cache()
# After an epoch, start_batch_id is set to zero
# non-zero value is only for the first epoch after loading pre-trained model
start_batch_id = 0
# save model for final step
self.save(self.checkpoint_dir, start_iters)
torch.cuda.empty_cache()
#caculat thresold
self.thresold()
