def forward(self, x, x_lengths, speaker_embedding, g=None):
x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h]
x = torch.transpose(x, 1, -1) # [b, h, t]
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
# Append speaker embeddings
speaker_embedding = speaker_embedding.unsqueeze(2)
speaker_embedding = speaker_embedding.repeat(1, 1, x.shape[2])
x = torch.cat((x, speaker_embedding), dim=1)
if self.prenet:
x = self.pre(x, x_mask)
x = self.encoder(x, x_mask)
if g is not None:
g_exp = g.expand(-1, -1, x.size(-1))
x_dp = torch.cat([torch.detach(x), g_exp], 1)
else:
x_dp = torch.detach(x)
x_m = self.proj_m(x) * x_mask
if not self.mean_only:
x_logs = self.proj_s(x) * x_mask
else:
x_logs = torch.zeros_like(x_m)
logw = self.proj_w(x_dp, x_mask)
return x_m, x_logs, logw, x_mask
def forward(self, x, x_lengths, g=None):
# embedding layer
# [B ,T, D]
x = self.emb(x) * math.sqrt(self.hidden_channels)
# [B, D, T]
x = torch.transpose(x, 1, -1)
# compute input sequence mask
x_mask = torch.unsqueeze(sequence_mask(x_lengths, x.size(2)),
1).to(x.dtype)
# pre-conv layers
if self.encoder_type in ['transformer', 'time-depth-separable']:
if self.use_prenet:
x = self.pre(x, x_mask)
# encoder
x = self.encoder(x, x_mask)
# set duration predictor input
if g is not None:
g_exp = g.expand(-1, -1, x.size(-1))
x_dp = torch.cat([torch.detach(x), g_exp], 1)
else:
x_dp = torch.detach(x)
# final projection layer
x_m = self.proj_m(x) * x_mask
if not self.mean_only:
x_logs = self.proj_s(x) * x_mask
else:
x_logs = torch.zeros_like(x_m)
# duration predictor
logw = self.duration_predictor(x_dp, x_mask)
return x_m, x_logs, logw, x_mask
def eval_cs_align_cost(self, S, XYZI_m, C, CC, CCs):
cost = 0
CCs = torch.detach(CCs)
CC = torch.detach(CC)
samp = S.reshape((self.batch_size, self.n_samples, 1, S.shape[-2], S.shape[-1]))
pre_s = C[:,0].reshape((self.batch_size, self.n_samples, 1, S.shape[-2], S.shape[-1]))
post_s = C[:,1].reshape((self.batch_size, self.n_samples, 1, S.shape[-2], S.shape[-1]))
pre_xyz = CC[:,0:3]
post_xyz = CC[:,4:7]
pre_xyzs = CCs[:,0:3]
post_xyzs = CCs[:,4:7]
mask_pre = (pre_s * samp).repeat_interleave(3,2)
mask_post = (post_s * samp).repeat_interleave(3,2)
dist_pre = torch.distributions.normal.Normal(pre_xyz,pre_xyzs)
dist_post = torch.distributions.normal.Normal(post_xyz,post_xyzs)
log_pre = dist_pre.log_prob(XYZI_m[:,:3])[:,None]
log_post = dist_post.log_prob(XYZI_m[:,:3])[:,None]
cost += (mask_pre* log_pre).sum(2)
cost += (mask_post * log_post).sum(2)
cost =-cost.sum(-1).sum(-1)
return cost
def forward(self, x, x_lengths, g=None):
x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h]
x = torch.transpose(x, 1, -1) # [b, h, t]
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),
1).to(x.dtype)
if self.prenet:
x = self.pre(x, x_mask)
x = self.encoder(x, x_mask)
if g is not None:
g_exp = g.expand(-1, -1, x.size(-1))
x_dp = torch.cat([torch.detach(x), g_exp], 1)
else:
x_dp = torch.detach(x)
x_m = self.proj_m(x) * x_mask
if not self.mean_only:
x_logs = self.proj_s(x) * x_mask
else:
x_logs = torch.zeros_like(x_m)
logw = self.proj_w(x_dp, x_mask)
x_pitch = self.proj_pitch(x_dp, x_mask)
x_pitch *= x_mask
return x_m, x_logs, logw, x_mask, x_pitch
def learn(self):
sample_index = np.random.choice(self.memory_capacity, self.batch_size)
batch_memory = self.replay_buffer[sample_index, :]
batch_state = torch.FloatTensor(batch_memory[:, :self.state_dim])
batch_action = torch.LongTensor(
batch_memory[:, self.state_dim:self.state_dim + 1].astype(int))
batch_reward = torch.FloatTensor(batch_memory[:, self.state_dim +
1:self.state_dim + 2])
batch_next_state = torch.FloatTensor(
batch_memory[:, self.state_dim + 2:2 * self.state_dim + 2])
batch_done = torch.FloatTensor(batch_memory[:, -1:])
next_value = self.target_model.forward(batch_next_state)
max_value = torch.max(next_value, dim=1)[0]
torch.detach(max_value)
target = batch_reward.squeeze() + self.gamma * (
1 - batch_done).squeeze() * max_value
q_value = self.model.forward(batch_state)
behavior = torch.gather(q_value, dim=1, index=batch_action).squeeze()
self.optimizer.zero_grad()
output = self.loss(behavior, target)
output.backward()
self.optimizer.step()
return output
def plot_bounday_torch(model,n_models,x_train,y_train,x_test,y_test):
w = model[-2].weight[0]
idx = np.where(w < 0)[0]
colors = ['black' if l == 0 else 'purple' for l in y_train]
plot_step = 0.02
x_min, x_max = x_train[:, 0].min() - 1, x_train[:, 0].max() + 1
y_min, y_max = x_train[:, 1].min() - 1, x_train[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
np.arange(y_min, y_max, plot_step))
input = np.c_[xx.ravel(), yy.ravel()]
Z = torch.from_numpy(input).float()
counter = 0
for layer in model:
Z = layer(Z)
if counter==3:
break
counter += 1
Z = torch.detach(Z)
n_rows = int(np.floor(np.sqrt(n_models)))
n_cols = int(np.floor(np.sqrt(n_models)))
fig, ax = plt.subplots(nrows=n_rows,ncols=n_cols)
model_number = 0
for r in range(n_rows):
for c in range(n_cols):
Z_current = np.round(np.reshape(Z[:,model_number], xx.shape))
if model_number in idx:
Z_current = 1 - Z_current
boundary = ax[r,c].contourf(xx, yy, Z_current, cmap=plt.cm.RdYlBu)
ax[r,c].scatter(x_train[:, 0], x_train[:, 1], c=colors,s=5)
model_number += 1
fig.colorbar(boundary, ax=ax, shrink=0.6)
ax[0, 0].set_title('Aggregation layer decision boundaries')
ax[0,1].set_title('Aggregation layer decision boundaries')
plt.axis('off')
plt.show(block=False)
plt.axis('off')
plt.show()
plt.figure()
input = np.c_[xx.ravel(), yy.ravel()]
Z = torch.from_numpy(input).float()
preds = torch.detach(model(Z))
preds = np.reshape(preds, xx.shape)
boundary = plt.contourf(xx, yy, preds, cmap=plt.cm.RdYlBu)
plt.scatter(x_train[:,0],x_train[:,1],c=y_train)
plt.show()
def clone_dist(self, dist, detach=False):
if self._rssm_type == 'discrete':
mean = dist.mean
if detach:
mean = th.detach(mean)
return td.Independent(OneHotDistFlattenSample(mean), 1)
else:
mean, stddev = dist.mean, dist.stddev
if detach:
mean, stddev = th.detach(mean), th.detach(stddev)
return td.Independent(td.Normal(mean, stddev), 1)
def forward(self, x, x_mask, g=None):
x = torch.detach(x)
if g is not None:
g = torch.detach(g)
x = x + self.cond(g)
x = self.conv_1(x * x_mask)
x = torch.relu(x)
x = self.norm_1(x)
x = self.drop(x)
x = self.conv_2(x * x_mask)
x = torch.relu(x)
x = self.norm_2(x)
x = self.drop(x)
x = self.proj(x * x_mask)
return x * x_mask
def encoder_pretrain_update(self, x, gt_vec):
assert self.ispretrain, "the method can be only used in pretrain mode"
self.encoder_opt.zero_grad()
# encode
_, _, param_encoded = self.model(x)
# get groundtruth
assert gt_vec.shape == param_encoded.shape # (bs,22)
# loss
param_gt = torch.detach(gt_vec).float()
self.loss_s = self.param_recon_criterion(param_encoded[:, 0],
param_gt[:, 0])
self.loss_t = self.param_recon_criterion(param_encoded[:, 1:3],
param_gt[:, 1:3])
self.loss_r = self.param_recon_criterion(param_encoded[:, 3:6],
param_gt[:, 3:6])
self.loss_pose = self.param_recon_criterion(param_encoded[:, 6:12],
param_gt[:, 6:12])
self.loss_beta = self.param_recon_criterion(param_encoded[:, 12:],
param_gt[:, 12:])
w = self.weight
self.vec_rec_loss = w.w_L1_pose * self.loss_pose + \
w.w_L1_beta * self.loss_beta + \
w.w_L1_r * self.loss_r + \
w.w_L1_t * self.loss_t + \
w.w_L1_s * self.loss_s
self.vec_rec_loss.backward()
self.encoder_opt.step()
def grad_scale(self, x, scale):
y_out = x
y_grad = x * scale
y = torch.detach(y_out - y_grad) + y_grad
return y
def forward(self, input, target):
N = target.size(0)
# print(N)
smooth = 1
input_flat = input.view(N, -1)
target_flat = target.view(N, -1)
input_np = torch.detach(input_flat).cpu().numpy()
input_np = preprocessing.Binarizer(threshold=0.5).transform(input_np)
input_flat = torch.from_numpy(input_np)
input_flat = input_flat.to(device)
# input_flat = torch.detach(input_flat).cpu().numpy()
# target_flat = torch.detach(target_flat).cpu().numpy()
# print(input_flat)
# print(target_flat)
# os.system("pause")
# print(input_flat.size())
intersection = input_flat * target_flat
# if target_flat.sum()==0:
# loss = input_flat.sum()/input_flat.sum()
# else:
# loss = 2 * (intersection.sum()) / (input_flat.sum() + target_flat.sum())
loss1 = 2 * intersection.sum()
loss2 = input_flat.sum() + target_flat.sum()
# loss = 1 - loss/ N
# loss = 2 * (intersection.sum(1)) / (input_flat.sum(1) + target_flat.sum(1)+smooth)
# loss = 1 - loss/ N
# loss =1-(intersection.sum(1)+smooth) / (target_flat.sum(1)+smooth)
# print(intersection.sum(1))
# print(input_flat.sum(1))
# print(target_flat.sum(1))
# os.system("pause")
return loss1, loss2
def show_model_latent(self):
device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
self.model: nn.Module = realNVP(self.hidden_layers, self.n_layers)
self.model.load_state_dict(
torch.load(checkpoint_fname, map_location=device))
self.model.to(device)
self.model.eval()
data_x, data_y = sample_data()
x_array = torch.from_numpy(data_x).float()
z, _ = self.model(x_array.to(device))
z = torch.detach(z.to('cpu')).numpy()
plt.scatter(z[:, 0][data_y == 0],
z[:, 1][data_y == 0],
c='r',
s=1,
label='0')
plt.scatter(z[:, 0][data_y == 1],
z[:, 1][data_y == 1],
c='b',
s=1,
label='1')
plt.scatter(z[:, 0][data_y == 2],
z[:, 1][data_y == 2],
c='g',
s=1,
label='2')
plt.legend()
plt.show()
pdb.set_trace()
def __getitem__(self, index):
img_path = self.file_path[index]
img = Image.open(img_path)
img_transform = self.transform(img)
img_transform = img_transform.unsqueeze(0)
img_encoder = self.encoder(img_transform)
img_encoder = torch.detach(img_encoder).numpy()
img_encoder = img_encoder.reshape(img_encoder.shape[1])
if 'apple' in img_path:
labels = 0
if 'banana' in img_path:
labels = 1
if 'grape' in img_path:
labels = 2
if 'mango' in img_path:
labels = 3
if 'orange' in img_path:
labels = 4
if 'pear' in img_path:
labels = 5
if 'pineapple' in img_path:
labels = 6
if 'tangerine' in img_path:
labels = 7
if 'tomato' in img_path:
labels = 8
if 'watermelon' in img_path:
labels = 9
return img_encoder, labels
def show_model_density(self, xrange, num=16, checkpoint_fname=None):
device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
self.model: nn.Module = realNVP(self.hidden_layers, self.n_layers)
self.model.load_state_dict(
torch.load(checkpoint_fname, map_location=device))
self.model.to(device)
self.model.eval()
x = np.linspace(xrange[0], xrange[1], num)
x0, x1 = np.meshgrid(x, x)
data_x = np.array([[x0[i, j], x1[i, j]] for i in range(num)
for j in range(num)])
x_array = torch.from_numpy(data_x).float()
z, y_arr = self.model(x_array.to(device))
y_arr = torch.detach(y_arr.to('cpu')).numpy()
y_arr = np.exp(y_arr)
y_arr = griddata(data_x, y_arr, (x0, x1), method='nearest')
fig = plt.figure(2)
ax = fig.gca()
ax.imshow(y_arr,
interpolation='gaussian',
origin='low',
extent=[x0.min(), x0.max(),
x1.min(), x1.max()])
plt.show()
def evaluate(self, _, __, edge_idx, edge_type):
with torch.no_grad():
self.eval()
n = edge_idx.shape[1]
if n > 15000:
step = n // 4
scores = []
for i in range(0, n, step):
batch_idx, batch_type = edge_idx[:, i:i + step], edge_type[i:i + step]
preds = torch.detach(self.forward(batch_idx, batch_type).view(-1).cpu())
scores.append(preds)
scores = torch.cat(scores)
else:
scores = torch.detach(self.forward(edge_idx, edge_type).view(-1).cpu())
return scores
def forward(self, scores):
js = js_fgan_lower_bound(scores)
if not self.clip:
dv = donsker_vardhan_lower_bound(scores)
else:
dv = smile_mi_lower_bound(scores, clamp=self.clip)
return js + torch.detach(dv - js)
def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
x = torch.detach(x)
x = self.pre(x)
if g is not None:
g = torch.detach(g)
x = x + self.cond(g)
x = self.convs(x, x_mask)
x = self.proj(x) * x_mask
if not reverse:
flows = self.flows
assert w is not None
logdet_tot_q = 0
h_w = self.post_pre(w)
h_w = self.post_convs(h_w, x_mask)
h_w = self.post_proj(h_w) * x_mask
e_q = torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype) * x_mask
z_q = e_q
for flow in self.post_flows:
z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
logdet_tot_q += logdet_q
z_u, z1 = torch.split(z_q, [1, 1], 1)
u = torch.sigmoid(z_u) * x_mask
z0 = (w - u) * x_mask
logdet_tot_q += torch.sum((F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1,2])
logq = torch.sum(-0.5 * (math.log(2*math.pi) + (e_q**2)) * x_mask, [1,2]) - logdet_tot_q
logdet_tot = 0
z0, logdet = self.log_flow(z0, x_mask)
logdet_tot += logdet
z = torch.cat([z0, z1], 1)
for flow in flows:
z, logdet = flow(z, x_mask, g=x, reverse=reverse)
logdet_tot = logdet_tot + logdet
nll = torch.sum(0.5 * (math.log(2*math.pi) + (z**2)) * x_mask, [1,2]) - logdet_tot
return nll + logq # [b]
else:
flows = list(reversed(self.flows))
flows = flows[:-2] + [flows[-1]] # remove a useless vflow
z = torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype) * noise_scale
for flow in flows:
z = flow(z, x_mask, g=x, reverse=reverse)
z0, z1 = torch.split(z, [1, 1], 1)
logw = z0
return logw
def encoder_update(self, x, gt_2d, gt_3d, mask, valid):
assert not self.ispretrain, "the method can be only used in train mode"
self.encoder_opt.zero_grad()
x, valid = torch.detach(x), torch.detach(valid)
gt_2d, gt_3d, mask = torch.detach(gt_2d), torch.detach(
gt_3d), torch.detach(mask)
# forward
x2d, x3d, param = self.model(x)
joint_2d, mesh_2d = x2d[:, :42], x2d[:, 42:]
joint_3d, mesh_3d = x3d[:, :21, :], x3d[:, 21:, :]
scale = param[:, 0]
trans = param[:, 1:3]
#test
#img = np.transpose(x[0].cpu().numpy()*255, axes=(1,2,0))
# show_line_on_img(img, gt_2d[0].cpu().numpy(), '/home/workspace2/dataset/3dhand/dataset/pts_on_img4.png')
# show_pts_on_img(img, self.convert_vec_to_2d(x2d)[0].detach().cpu().numpy(), '/home/workspace2/dataset/3dhand/dataset/mesh_on_img5.png')
# show_line_on_img(img, self.convert_vec_to_2d(joint_2d)[0].detach().cpu().numpy(), '/home/workspace2/dataset/3dhand/dataset/pts_on_img5.png')
# show_mask_on_img(img, mask[0].detach().cpu().numpy())
#show_3dmesh(x3d[0])
self.loss_2d = self.comupte_2d_joint_loss(joint_2d, gt_2d)
if '3d_norm' in self.weight.values(
) or '3d_norm_no_detach' in self.weight.values():
self.loss_3d = self.compute_3d_joint_loss_norm(
joint_3d,
gt_3d,
valid[:, 0],
detach='3d_norm_no_detach' not in self.weight.values())
else:
self.loss_3d = self.compute_3d_joint_loss(joint_3d, gt_3d,
valid[:, 0], scale)
self.loss_mask = self.compute_mask_loss(mesh_2d, mask, valid[:, 1])
self.loss_reg = self.compute_param_reg_loss(param)
w = self.weight
self.train_loss = w.w_2d * self.loss_2d + \
w.w_3d * self.loss_3d + \
w.w_mask * self.loss_mask + \
w.w_reg * self.loss_reg
self.train_loss.backward()
self.encoder_opt.step()
def sample_train(self, x, gt_2d, gt_3d, mask, valid):
assert not self.ispretrain, "the method can be only used in train mode"
x, valid = torch.detach(x), torch.detach(valid)
gt_2d, gt_3d, mask = torch.detach(gt_2d), torch.detach(
gt_3d), torch.detach(mask)
# forward
x2d, x3d, param = self.model(x)
scale = param[:, 0]
joint_2d, mesh_2d = x2d[:, :42], x2d[:, 42:]
joint_3d, mesh_3d = x3d[:, :21, :], x3d[:, 21:, :]
self.loss_2d = self.comupte_2d_joint_loss(joint_2d, gt_2d)
if '3d_norm' in self.weight.values(
) or '3d_norm_no_detach' in self.weight.values():
self.loss_3d = self.compute_3d_joint_loss_norm(
joint_3d,
gt_3d,
valid[:, 0],
detach='3d_norm_no_detach' not in self.weight.values())
else:
self.loss_3d = self.compute_3d_joint_loss(joint_3d, gt_3d,
valid[:, 0], scale)
self.loss_mask = self.compute_mask_loss(mesh_2d, mask, valid[:, 1])
self.loss_reg = self.compute_param_reg_loss(param)
w = self.weight
self.train_loss = w.w_2d * self.loss_2d + \
w.w_3d * self.loss_3d + \
w.w_mask * self.loss_mask + \
w.w_reg * self.loss_reg
losses = np.array([
self.train_loss.cpu().numpy(),
self.loss_2d.cpu().numpy(),
self.loss_3d.cpu().numpy(),
self.loss_mask.cpu().numpy(),
self.loss_reg.cpu().numpy()
])
return [joint_2d, mesh_2d, joint_3d, mesh_3d], losses
def forward(self, *, text, text_lengths, speaker_embeddings=None):
x = self.emb(text) * math.sqrt(self.hidden_channels) # [b, t, h]
x = torch.transpose(x, 1, -1) # [b, h, t]
x_mask = torch.unsqueeze(
glow_tts_submodules.sequence_mask(text_lengths, x.size(2)),
1).to(x.dtype)
if self.prenet:
x = self.pre(x, x_mask)
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
for i in range(self.n_layers):
x = x * x_mask
y = self.attn_layers[i](x, x, attn_mask)
y = self.drop(y)
x = self.norm_layers_1[i](x + y)
y = self.ffn_layers[i](x, x_mask)
y = self.drop(y)
x = self.norm_layers_2[i](x + y)
x = x * x_mask
if speaker_embeddings is not None:
g_exp = speaker_embeddings.expand(-1, -1, x.size(-1))
x_dp = torch.cat([torch.detach(x), g_exp], 1)
else:
x_dp = torch.detach(x)
x_m = self.proj_m(x) * x_mask
if not self.mean_only:
x_logs = self.proj_s(x) * x_mask
else:
x_logs = torch.zeros_like(x_m)
logw = self.proj_w(spect=x_dp, mask=x_mask)
return x_m, x_logs, logw, x_mask
def forward(self, x, x_lengths, g=None):
"""
Shapes:
- x: :math:`[B, C, T]`
- x_lengths: :math:`[B]`
- g (optional): :math:`[B, 1, T]`
"""
# embedding layer
# [B ,T, D]
x = self.emb(x) * math.sqrt(self.hidden_channels)
# [B, D, T]
x = torch.transpose(x, 1, -1)
# compute input sequence mask
x_mask = torch.unsqueeze(sequence_mask(x_lengths, x.size(2)),
1).to(x.dtype)
# prenet
if hasattr(self, "prenet") and self.use_prenet:
x = self.prenet(x, x_mask)
# encoder
x = self.encoder(x, x_mask)
# postnet
if hasattr(self, "postnet"):
x = self.postnet(x) * x_mask
# set duration predictor input
if g is not None:
g_exp = g.expand(-1, -1, x.size(-1))
x_dp = torch.cat([torch.detach(x), g_exp], 1)
else:
x_dp = torch.detach(x)
# final projection layer
x_m = self.proj_m(x) * x_mask
if not self.mean_only:
x_logs = self.proj_s(x) * x_mask
else:
x_logs = torch.zeros_like(x_m)
# duration predictor
logw = self.duration_predictor(x_dp, x_mask)
return x_m, x_logs, logw, x_mask
def get_cross_entropy_loss(self,
input,
target,
is_expanded_target=True,
need_PR=False):
if is_expanded_target:
target = torch.argmax(target, dim=1)
logits = self._get_logits(input)
if need_PR:
return F.cross_entropy(logits,
target), F.softmax(torch.detach(logits),
dim=1)
else:
return F.cross_entropy(logits, target)
def forward(self, input):
en_res = self.encoder(input)
en_res = en_res.permute(0, 2, 3, 1)
# print(en_res.shape)
fc_res1 = self.channel_fc1(en_res)
fc_res1 = fc_res1.permute(0, 2, 1, 3)
# print(fc_res1.shape)
fc_res2 = self.channel_fc2(fc_res1)
fc_res = fc_res2.permute(0, 3, 2, 1)
de_res = self.decoder(fc_res)
tmp = torch.detach(de_res)
if np.any(np.isnan(tmp.cpu().numpy()) == True):
exit()
return de_res
def func_and_grad(self):
#print('Jacobian calculating is started')
start_time = time.time()
self.set_indices()
if self.defaults['x_in'] is None:
params = self.get_params(self.param_groups)
#print('params = {}'.format(*params))
#print('junc = {}'.format(self.defaults['indices']))
F = self.defaults['function'](*params)[self.defaults['indices']]
else:
F = (self.defaults['function'](self.defaults['x_in']) -
self.defaults['y_target'])[self.defaults['indices']].reshape(
-1, 1)
#print('x : {} F : {}'.format(self.defaults['x_in'].shape, F.shape))
grad_F = []
#print('F = {}'.format(F))
size = F.shape[0]
for i, F_i in enumerate(F):
#print('{} of {}'.format(i, size))
#print('F_i = {} done\n\n'.format(F_i))
backward(F_i, retain_graph=True)
grad_groups = []
for group in self.param_groups:
grad_group = []
for p in group['params']:
grad_group.append(p.grad.data)
p.grad.data = torch.zeros_like(p.grad.data)
grad_groups.append(grad_group)
grad_F.append(grad_groups)
F = detach(F)
#print('Jacobian calculating time = {}'.format(time.time() - start_time))
return F, grad_F
def kmeans_predict(self, n_class, y_s, x_s, x_u, x_q, vis, cm):
"""
x_s: [B, Ns, 2048, 1]
x_u: [B, Nu, 2048, 1]
x_q: [B, Nq, 2048, 1]
y_s: [B, Ns, 1]
:returns: logits [B, N, K]
"""
h_s, h_u, h_q = self.get_encoded_inputs(x_s, x_u,
x_q) # 3个均为 [B, N, D]
y_s = y_s.squeeze(dim=-1)
# if cm:
# data = h_q.detach().reshape(-1, h_q.shape[-1]).cpu().numpy() # (nc*nq, z_dim)
# label = y_s.cpu().numpy().reshape(-1) # (n,)
# vis_tSNE(data, label, classes=n_class, vis=vis, name='query after', n_dim=2)
# t_sne(data, label, classes=n_class, name='query after', n_dim=2)
# # [nc, z_dim]==>[nc, 1, z_dim]==>[nc, ns, z_dim]
# 使用.expand似乎会造成存储空间不连续，类似于对tensor实施转置也会造成存储空间不连续
# 此时，就不能使用.view()，解决方法是在.view()前增加.contiguous()
protos = self.compute_protos(h_s) # [K, D] 即 [Nc, D]
# Hard assignment for training images.
prob_s = [None] * n_class
for kk in range(n_class):
prob_s[kk] = torch.unsqueeze(torch.eq(y_s, kk).float(),
dim=2) # [B, N, 1]
prob_s = torch.cat(prob_s,
dim=2) # [B, N, K] the probability of support_data
h_all = torch.cat([h_s, h_u], dim=1) # [B, Ns+Nu, D]
# Run clustering.
for tt in range(self.num_cluster_steps):
prob_u = assign_cluster(protos, h_u) # [B, P, K]
prob_all = torch.cat([prob_s, prob_u], dim=1) # [B, N+P, K]
prob_all = torch.detach(prob_all)
# ##################################问题所在############################
# protos = protos + update_cluster(h_all, prob_all) # [K, D]
protos = update_cluster(h_all, prob_all) # [K, D]
logits = compute_logits(protos, h_q) # [B, N, K]
return logits, h_q
def get_propability_map(cv, depth_map, depth_start, depth_interval):
"""get probability map from cost volume"""
with torch.no_grad():
batch_size, channels, height, width = list(depth_map.size())
depth = cv.size(1)
# byx coordinates, batched & flattened
b_coordinates = torch.arange(batch_size, dtype=torch.int64)
y_coordinates = torch.arange(height, dtype=torch.int64)
x_coordinates = torch.arange(width, dtype=torch.int64)
b_coordinates = b_coordinates.view(batch_size, 1,
1).expand(batch_size, height, width)
y_coordinates = y_coordinates.view(1, height,
1).expand(batch_size, height, width)
x_coordinates = x_coordinates.view(1, 1, width).expand(
batch_size, height, width)
b_coordinates = b_coordinates.contiguous().view(-1).type(torch.long)
y_coordinates = y_coordinates.contiguous().view(-1).type(torch.long)
x_coordinates = x_coordinates.contiguous().view(-1).type(torch.long)
# b_coordinates = _repeat_(b_coordinates, batch_size)
# y_coordinates = _repeat_(y_coordinates, batch_size)
# x_coordinates = _repeat_(x_coordinates, batch_size)
# d coordinates (floored and ceiled), batched & flattened
d_coordinates = ((depth_map - depth_start.view(-1, 1, 1, 1)) /
depth_interval.view(-1, 1, 1, 1)).view(-1)
d_coordinates = torch.detach(d_coordinates)
d_coordinates_left0 = torch.clamp(d_coordinates.floor(), 0,
depth - 1).type(torch.long)
d_coordinates_right0 = torch.clamp(d_coordinates.ceil(), 0,
depth - 1).type(torch.long)
# # get probability image by gathering
prob_map_left0 = cv[b_coordinates, d_coordinates_left0, y_coordinates,
x_coordinates]
prob_map_right0 = cv[b_coordinates, d_coordinates_right0,
y_coordinates, x_coordinates]
prob_map = prob_map_left0 + prob_map_right0
prob_map = prob_map.view(batch_size, 1, height, width)
return prob_map
def numpy_tensor_serializer(worker: AbstractWorker,
tensor: torch.Tensor) -> bin:
"""Strategy to serialize a tensor using numpy npy format.
If tensor requires to calculate gradients, it will be detached.
Args
(torch.Tensor): an input tensor to be serialized
Returns
A serialized version of the input tensor
"""
if tensor.requires_grad:
warnings.warn(
"Torch to Numpy serializer can only be used with tensors that do not require grad. "
"Detaching tensor to continue")
tensor = torch.detach()
np_tensor = tensor.numpy()
outfile = io.BytesIO()
numpy.save(outfile, np_tensor)
return outfile.getvalue()
def show_latent(k, fpath):
nin = 2
nout = 3 * k * nin
model: nn.Module = Model(nin, [20, 20, 20], nout, natural_ordering=True, bias_init=1.0)
model.load_state_dict(torch.load(fpath))
data_x, data_y = sample_data()
train_x, test_x, train_y, test_y = divide_ds(data_x, data_y, 0.8)
train_x = torch.from_numpy(train_x)
test_x = torch.from_numpy(test_x)
_, _, z_var = model(test_x)
z_var = torch.detach(z_var.to('cpu')).numpy()
colors = ['r', 'b', 'g']
for i in range(3):
color = colors[i]
z = z_var[test_y == i, :]
pdb.set_trace()
def attack(self, model, x, y, target=False):
self.model_ = model
self.min_ = 0.0
self.max_ = 1.0
self.noise_ = torch.zeros(x.size())
success = False
n_queries = 0
adv = None
y_hat = None
for i in range(self.n_iter):
adv = x + self.noise_
adv = torch.clamp(adv, self.min_, self.max_)
adv.requires_grad = True
y_hat = self.model_(adv)
n_queries += 1
if target:
success = torch.argmax(y_hat).item() == y.item()
else:
success = torch.argmax(y_hat).item() != y.item()
if success:
adv = torch.detach(adv)
adv = torch.squeeze(adv)
break
model.zero_grad()
loss = F.cross_entropy(y_hat, y)
loss.backward()
if target:
self.noise_ -= self.step_size * torch.sign(adv.grad.data)
else:
self.noise_ += self.step_size * torch.sign(adv.grad.data)
self.noise_ = torch.clamp(self.noise_, -self.epsilon, self.epsilon)
return success, n_queries, adv, y_hat
def show_model_density(k, fpath, hidden_layers):
nin = 2
nout = 3 * k * nin
model: nn.Module = Model(nin, hidden_layers, nout, natural_ordering=True, bias_init=1.0)
model.load_state_dict(torch.load(fpath))
num = 16
x = np.linspace(-4, 4, num)
x0, x1 = np.meshgrid(x, x)
# x_array = np.transpose([np.tile(x, len(x)), np.repeat(x, len(x))])
x_array = np.array([[x0[i, j], x1[i, j]] for i in range(num) for j in range(num)])
_ , y_arr, _ = model(torch.from_numpy(x_array))
y_arr = torch.detach(y_arr.to('cpu')).numpy()
y_arr = np.reshape(y_arr, (num, num))
y_arr = np.exp(y_arr)
fig = plt.figure(2)
ax = fig.gca()
ax.imshow(y_arr, interpolation='gaussian', origin='low',
extent=[x0.min(), x0.max(), x1.min(), x1.max()])
plt.show()
