def barycentric_model_batch_loss(self, u_batch, v_batch, Xs_batch, Xt_batch, fXs_batch):
C_batch = self.computeSquareEuclideanCostMatrix(Xs_batch, Xt_batch)
if self.reg_type == 'entropy':
H = torch.exp((torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch) / self.reg_val)
elif self.reg_type == 'l2':
H = (1./(2.*self.reg_val))*torch.max(torch.zeros(1, dtype=self.dtype, device=self.device), (torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch))
d_batch = self.computeSquareEuclideanCostMatrix(fXs_batch, Xt_batch)
return torch.sum(torch.mul(d_batch, H))
def dual_OT_batch_loss(self, batch_size, u_batch, v_batch, Xs_batch, Xt_batch):
C_batch = self.computeSquareEuclideanCostMatrix(Xs_batch=Xs_batch, Xt_batch=Xt_batch)
if self.reg_type == 'entropy':
loss_batch = torch.sum(u_batch)*batch_size + torch.sum(v_batch)*batch_size \
- self.reg_val*torch.sum(torch.exp((torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch) / self.reg_val))
elif self.reg_type == 'l2':
tmp = torch.max(torch.zeros(1, dtype=self.dtype, device=self.device), (torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch))
loss_batch = torch.sum(u_batch)*batch_size + torch.sum(v_batch)*batch_size \
- (1./(4.*self.reg_val))*torch.sum(torch.mul(tmp, tmp))
return -loss_batch
def forward(self, X, X_mask):
#X_mask not used
#X: [m, V, Tx] m = batch size, V = vocabulary size, Tx = char count (max 1014)
#print(X.size(), type(X))
m = X.size()[0]
V = X.size()[1]
Tx = X.size()[2]
#conv layer
for module in self.convlayers:
#print(X.size(), type(X.data))
if 'Conv1d' in str(type(module)): X = X.unsqueeze(1)
X = module(X)
if 'Conv1d' in str(type(module)): X = X.squeeze()
#X: [m, num_filters, 14]
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, self.num_filters, 14])
#linear
X = torch.reshape(X, (m, 14*self.num_filters))
assert X.size() == torch.Size([m, self.num_filters*14])
X = self.linear(X)
#X: [m, 1]
#print(X.size(), type(X))
if self.num_classes == 2:
assert X.size() == torch.Size([m, 1])
else:
assert X.size() == torch.Size([m, self.num_classes])
if self.num_classes == 2:
X = torch.squeeze(X)
X = self.sigmoid(X)
#X: [m]
#print(X.size(), type(X))
assert X.size() == torch.Size([m])
return X
else:
return F.softmax(X)
def preprocessing_fn_torch(batch,
consecutive_frames=16,
scale_first=True,
align_corners=False):
"""
inputs - batch of videos each with shape (frames, height, width, channel)
outputs - batch of videos each with shape (n_stack, channel, stack_frames, new_height, new_width)
frames = n_stack * stack_frames (after padding)
new_height = new_width = 112
consecutive_frames - number of consecutive frames (stack_frames)
After resizing, a center crop is performed to make the image square
This is a differentiable alternative to MARS' PIL-based preprocessing.
There are some
"""
if not isinstance(batch, torch.Tensor):
logger.warning(f"batch {type(batch)} is not a torch.Tensor. Casting")
batch = torch.from_numpy(batch).to(DEVICE)
# raise ValueError(f"batch {type(batch)} is not a torch.Tensor")
if batch.dtype != torch.float32:
raise ValueError(f"batch {batch.dtype} should be torch.float32")
if batch.shape[0] != 1:
raise ValueError(f"Batch size {batch.shape[0]} != 1")
video = batch[0]
if video.ndim != 4:
raise ValueError(
f"video dims {video.ndim} != 4 (frames, height, width, channel)")
if video.shape[0] < 1:
raise ValueError("video must have at least one frame")
if tuple(video.shape[1:]) != (240, 320, 3):
raise ValueError(
f"frame shape {tuple(video.shape[1:])} != (240, 320, 3)")
if video.max() > 1.0 or video.min() < 0.0:
raise ValueError("input should be float32 in [0, 1] range")
if not isinstance(consecutive_frames, int):
raise ValueError(
f"consecutive_frames {consecutive_frames} must be an int")
if consecutive_frames < 1:
raise ValueError(
f"consecutive_frames {consecutive_frames} must be positive")
# Select a integer multiple of consecutive frames
while len(video) < consecutive_frames:
# cyclic pad if insufficient for a single stack
video = torch.cat([video, video[:consecutive_frames - len(video)]])
if len(video) % consecutive_frames != 0:
# cut trailing frames
video = video[:len(video) - (len(video) % consecutive_frames)]
if scale_first:
# Attempts to directly follow MARS approach
# (frames, height, width, channel) to (frames, channel, height, width)
video = video.permute(0, 3, 1, 2)
sample_width, sample_height = 149, 112
video = torch.nn.functional.interpolate(
video,
size=(sample_height, sample_width),
mode="bilinear",
align_corners=align_corners,
)
crop_left = 18 # round((149 - 112)/2.0)
video = video[:, :, :, crop_left:crop_left + sample_height]
else:
# More efficient, but not MARS approach
# Center crop
sample_size = 112
upsample, downsample = 7, 15
assert video.shape[1] * upsample / downsample == sample_size
crop_width = 40
assert crop_width == (video.shape[2] - video.shape[1]) / 2
assert video.shape[1] + 2 * crop_width == video.shape[2]
video = video[:, :, crop_width:video.shape[2] - crop_width, :]
assert video.shape[1] == video.shape[2] == 240
# Downsample to (112, 112) frame size
# (frames, height, width, channel) to (frames, channel, height, width)
video = video.permute(0, 3, 1, 2)
video = torch.nn.functional.interpolate(
video,
size=(sample_size, sample_size),
mode="bilinear",
align_corners=align_corners,
)
if video.max() > 1.0:
raise ValueError("Video exceeded max after interpolation")
if video.min() < 0.0:
raise ValueError("Video under min after interpolation")
# reshape into stacks of frames
video = torch.reshape(video, (-1, consecutive_frames) + video.shape[1:])
# transpose to (stacks, channel, stack_frames, height, width)
video = video.permute(0, 2, 1, 3, 4)
# video = torch.transpose(video, axes=(0, 4, 1, 2, 3))
# normalize before changing channel position?
video = torch.transpose(video, 1, 4)
video = (
(video * 255) -
torch.from_numpy(MEAN).to(DEVICE)) / torch.from_numpy(STD).to(DEVICE)
video = torch.transpose(video, 4, 1)
return video
def pytorch_compute_single(im, target_interface, GPU_mode=1):
input_pytorch_value = None
output_pytorch_value = None
input_pytorch_cpu_value = None
output_pytorch_cpu_value = None
pytorch_shape = [1]
for shape_element in im.shape:
pytorch_shape.append(shape_element)
input_pytorch = torch.reshape(torch.from_numpy(im), pytorch_shape)
if GPU_mode != 0:
input_pytorch_cpu_value = input_pytorch.numpy()
if target_interface == 'conv1':
weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1)))
output_pytorch_cpu = torch.from_numpy(
F.conv2d(torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2))), weights_torch, padding=0,
stride=4).numpy().transpose((0, 2, 3, 1)))
elif target_interface == 'conv2':
x_torch = torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2)).astype(np.float64))
weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1)))
stride = 4
if x_torch.numpy().shape[2] % stride == 0:
pad = max(weights_torch.numpy().shape[2] - stride, 0)
else:
pad = max(weights_torch.numpy().shape[2] - (x_torch.numpy().shape[2] % stride), 0)
if pad % 2 == 0:
pad_val = pad // 2
padding = (pad_val, pad_val, pad_val, pad_val)
else:
pad_val_start = pad // 2
pad_val_end = pad - pad_val_start
padding = (pad_val_start, pad_val_end, pad_val_start, pad_val_end)
x_torch = F.pad(x_torch, padding, "constant", 0)
output_pytorch_cpu = torch.from_numpy(
F.conv2d(x_torch, weights_torch, padding=0, stride=stride).numpy().transpose((0, 2, 3, 1)))
elif target_interface == 'pool1':
output_pytorch_cpu = torch.from_numpy(np.rollaxis(
F.max_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_cpu_value, 3, 1)), kernel_size=(2, 2),
stride=(2, 2)).numpy(), 1, 4))
elif target_interface == 'pool2':
output_pytorch_cpu = torch.from_numpy(np.rollaxis(
F.avg_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_cpu_value, 3, 1)), kernel_size=(2, 2),
stride=(2, 2)).numpy(), 1, 4))
elif target_interface == 'relu1':
output_pytorch_cpu = F.relu(input_pytorch)
elif target_interface == 'dense1':
output_pytorch_cpu = None
elif target_interface == 'sigmoid1':
output_pytorch_cpu = F.sigmoid(input_pytorch)
elif target_interface == 'tanh1':
output_pytorch_cpu = F.tanh(input_pytorch)
else:
output_pytorch_cpu = None
output_pytorch_cpu_value = output_pytorch_cpu.numpy()
if GPU_mode != 2:
input_pytorch_value = input_pytorch.to("cuda").to("cpu").numpy()
if target_interface == 'conv1':
weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1)))
output_pytorch = torch.from_numpy(
F.conv2d(torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2))).to("cuda"),
weights_torch.to("cuda"), padding=0, stride=4).to("cpu").numpy().transpose((0, 2, 3, 1))).to(
"cuda")
elif target_interface == 'conv2':
x_torch = torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2)).astype(np.float64))
weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1)))
stride = 4
if x_torch.numpy().shape[2] % stride == 0:
pad = max(weights_torch.numpy().shape[2] - stride, 0)
else:
pad = max(weights_torch.numpy().shape[2] - (x_torch.numpy().shape[2] % stride), 0)
if pad % 2 == 0:
pad_val = pad // 2
padding = (pad_val, pad_val, pad_val, pad_val)
else:
pad_val_start = pad // 2
pad_val_end = pad - pad_val_start
padding = (pad_val_start, pad_val_end, pad_val_start, pad_val_end)
x_torch = F.pad(x_torch, padding, "constant", 0)
output_pytorch = torch.from_numpy(
F.conv2d(x_torch.to("cuda"), weights_torch.to("cuda"), padding=0, stride=4).to("cpu").numpy().transpose(
(0, 2, 3, 1))).to("cuda")
elif target_interface == 'pool1':
output_pytorch = torch.from_numpy(np.rollaxis(
F.max_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_value, 3, 1)).to("cuda"), kernel_size=(2, 2),
stride=(2, 2)).to("cpu").numpy(), 1, 4)).to("cuda")
elif target_interface == 'pool2':
output_pytorch = torch.from_numpy(np.rollaxis(
F.avg_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_value, 3, 1)).to("cuda"), kernel_size=(2, 2),
stride=(2, 2)).to("cpu").numpy(), 1, 4)).to("cuda")
elif target_interface == 'relu1':
output_pytorch = F.relu(input_pytorch.to("cuda"))
elif target_interface == 'dense1':
output_pytorch = None
elif target_interface == 'sigmoid1':
output_pytorch = F.sigmoid(input_pytorch.to("cuda"))
elif target_interface == 'tanh1':
output_pytorch = F.tanh(input_pytorch.to("cuda"))
else:
output_pytorch = None
output_pytorch_value = output_pytorch.to("cpu").numpy()
return output_pytorch_value, input_pytorch_value, output_pytorch_cpu_value, input_pytorch_cpu_value
def batch_global_rigid_transformation(Rs,
Js,
parent,
rotate_base=False,
betas_logscale=None,
opts=None):
"""
Computes absolute joint locations given pose.
rotate_base: if True, rotates the global rotation by 90 deg in x axis.
if False, this is the original SMPL coordinate.
Args:
Rs: N x 24 x 3 x 3 rotation vector of K joints
Js: N x 24 x 3, joint locations before posing
parent: 24 holding the parent id for each index
Returns
new_J : `Tensor`: N x 24 x 3 location of absolute joints
A : `Tensor`: N x 24 4 x 4 relative joint transformations for LBS.
"""
if rotate_base:
print('Flipping the SMPL coordinate frame!!!!')
rot_x = torch.Tensor([[1, 0, 0], [0, -1, 0], [0, 0, -1]])
rot_x = torch.reshape(torch.repeat(rot_x, [N, 1]),
[N, 3, 3]) # In tf it was tile
root_rotation = torch.matmul(Rs[:, 0, :, :], rot_x)
else:
root_rotation = Rs[:, 0, :, :]
# Now Js is N x 24 x 3 x 1
Js = Js.unsqueeze(-1)
N = Rs.shape[0]
Js_orig = Js.clone()
scaling_factors = torch.ones(N, parent.shape[0], 3).to(Rs.device)
if betas_logscale is not None:
leg_joints = list(range(7, 11)) + list(range(11, 15)) + list(
range(17, 21)) + list(range(21, 25))
tail_joints = list(range(25, 32))
ear_joints = [33, 34]
beta_scale_mask = torch.zeros(35, 3, 6).to(betas_logscale.device)
beta_scale_mask[leg_joints, [2], [0]] = 1.0 # Leg lengthening
beta_scale_mask[leg_joints, [0], [1]] = 1.0 # Leg fatness
beta_scale_mask[leg_joints, [1], [1]] = 1.0 # Leg fatness
beta_scale_mask[tail_joints, [0], [2]] = 1.0 # Tail lengthening
beta_scale_mask[tail_joints, [1], [3]] = 1.0 # Tail fatness
beta_scale_mask[tail_joints, [2], [3]] = 1.0 # Tail fatness
beta_scale_mask[ear_joints, [1], [4]] = 1.0 # Ear y
beta_scale_mask[ear_joints, [2], [5]] = 1.0 # Ear z
beta_scale_mask = torch.transpose(beta_scale_mask.reshape(35 * 3, 6),
0, 1)
betas_scale = torch.exp(betas_logscale @ beta_scale_mask)
scaling_factors = betas_scale.reshape(-1, 35, 3)
scale_factors_3x3 = torch.diag_embed(scaling_factors, dim1=-2, dim2=-1)
def make_A(R, t):
# Rs is N x 3 x 3, ts is N x 3 x 1
R_homo = torch.nn.functional.pad(R, (0, 0, 0, 1, 0, 0))
t_homo = torch.cat([t, torch.ones([N, 1, 1]).to(Rs.device)], 1)
return torch.cat([R_homo, t_homo], 2)
A0 = make_A(root_rotation, Js[:, 0])
results = [A0]
for i in range(1, parent.shape[0]):
j_here = Js[:, i] - Js[:, parent[i]]
s_par_inv = torch.inverse(scale_factors_3x3[:, parent[i]])
rot = Rs[:, i]
s = scale_factors_3x3[:, i]
rot_new = s_par_inv @ rot @ s
A_here = make_A(rot_new, j_here)
res_here = torch.matmul(results[parent[i]], A_here)
results.append(res_here)
# 10 x 24 x 4 x 4
results = torch.stack(results, dim=1)
# scale updates
new_J = results[:, :, :3, 3]
# --- Compute relative A: Skinning is based on
# how much the bone moved (not the final location of the bone)
# but (final_bone - init_bone)
# ---
Js_w0 = torch.cat([Js_orig, torch.zeros([N, 35, 1, 1]).to(Rs.device)], 2)
init_bone = torch.matmul(results, Js_w0)
# Append empty 4 x 3:
init_bone = torch.nn.functional.pad(init_bone, (3, 0, 0, 0, 0, 0, 0, 0))
A = results - init_bone
return new_J, A
def forward(self, graph, feat, weight=None):
r"""Compute graph convolution.
Notes
-----
* Input shape: :math:`(N, *, \text{in_feats})` where * means any number of additional
dimensions, :math:`N` is the number of nodes.
* Output shape: :math:`(N, *, \text{out_feats})` where all but the last dimension are
the same shape as the input.
* Weight shape: :math:`(\text{in_feats}, \text{out_feats})`.
Parameters
----------
graph : DGLGraph
The graph.
feat : torch.Tensor
The input feature
weight : torch.Tensor, optional
Optional external weight tensor.
Returns
-------
torch.Tensor
The output feature
"""
with graph.local_scope():
if self._norm == 'both':
degs = graph.out_degrees().to(feat.device).float().clamp(min=1)
norm = th.pow(degs, -0.5)
shp = norm.shape + (1, ) * (feat.dim() - 1)
norm = th.reshape(norm, shp)
feat = feat * norm
weight = self.weight
if self._in_feats > self._out_feats and False:
# mult W first to reduce the feature size for aggregation.
if weight is not None:
assert feat.shape[-1] == weight.shape[
-2], "feat shape is {}, weight shape is {}".format(
feat.shape, weight.shape)
feat = th.matmul(feat, weight)
feat = self.apply_allgather(feat)
graph.srcdata['h'] = feat
graph.update_all(fn.copy_src(src='h', out='m'),
fn.sum(msg='m', out='h'))
rst = graph.dstdata['h']
rst = rst[:self._local_n_nodes, :]
else:
# aggregate first then mult W
feat = self.apply_allgather(feat)
graph.srcdata['h'] = feat
graph.update_all(fn.copy_src(src='h', out='m'),
fn.sum(msg='m', out='h'))
rst = graph.dstdata['h']
rst = rst[:self._local_n_nodes, :]
if weight is not None:
rst = th.matmul(rst, weight)
if self._norm != 'none':
degs = graph.in_degrees().to(feat.device).float().clamp(min=1)
if self._norm == 'both':
norm = th.pow(degs, -0.5)
else:
norm = 1.0 / degs
shp = norm.shape + (1, ) * (feat.dim() - 1)
norm = th.reshape(norm, shp)
rst = rst * norm
if self.bias is not None:
rst = rst + self.bias
if self._comm_net:
rst += th.matmul(feat[:self._local_n_nodes], self.comm_net_w)
if self._activation is not None:
rst = self._activation(rst)
return rst
def forward(self, batch_data):
images_in, boxes_in, bboxes_num_in = batch_data
# read config parameters
B = images_in.shape[0]
T = images_in.shape[1]
H, W = self.cfg.image_size
OH, OW = self.cfg.out_size
MAX_N = self.cfg.num_boxes
NFB = self.cfg.num_features_boxes
NFR, NFG = self.cfg.num_features_relation, self.cfg.num_features_gcn
D = self.cfg.emb_features
K = self.cfg.crop_size[0]
if not self.training:
B = B * 3
T = T // 3
images_in.reshape((B, T) + images_in.shape[2:])
boxes_in.reshape((B, T) + boxes_in.shape[2:])
bboxes_num_in.reshape((B, T))
# Reshape the input data
images_in_flat = torch.reshape(images_in,
(B * T, 3, H, W)) #B*T, 3, H, W
boxes_in = boxes_in.reshape(B * T, MAX_N, 4)
# Use backbone to extract features of images_in
# Pre-precess first
images_in_flat = prep_images(images_in_flat)
outputs = self.backbone(images_in_flat)
# Build multiscale features
features_multiscale = []
for features in outputs:
if features.shape[2:4] != torch.Size([OH, OW]):
features = F.interpolate(features,
size=(OH, OW),
mode='bilinear',
align_corners=True)
features_multiscale.append(features)
features_multiscale = torch.cat(features_multiscale,
dim=1) #B*T, D, OH, OW
boxes_in_flat = torch.reshape(boxes_in,
(B * T * MAX_N, 4)) #B*T*MAX_N, 4
boxes_idx = [
i * torch.ones(MAX_N, dtype=torch.int) for i in range(B * T)
]
boxes_idx = torch.stack(boxes_idx).to(
device=boxes_in.device) # B*T, MAX_N
boxes_idx_flat = torch.reshape(boxes_idx,
(B * T * MAX_N, )) #B*T*MAX_N,
# RoI Align
boxes_in_flat.requires_grad = False
boxes_idx_flat.requires_grad = False
boxes_features_all = self.roi_align(
features_multiscale, boxes_in_flat,
boxes_idx_flat) #B*T*MAX_N, D, K, K,
boxes_features_all = boxes_features_all.reshape(B * T, MAX_N,
-1) #B*T,MAX_N, D*K*K
# Embedding
boxes_features_all = self.fc_emb_1(
boxes_features_all) # B*T,MAX_N, NFB
boxes_features_all = self.nl_emb_1(boxes_features_all)
boxes_features_all = F.relu(boxes_features_all)
boxes_features_all = boxes_features_all.reshape(B, T, MAX_N, NFB)
boxes_in = boxes_in.reshape(B, T, MAX_N, 4)
actions_scores = []
activities_scores = []
bboxes_num_in = bboxes_num_in.reshape(B, T) #B,T,
for b in range(B):
N = bboxes_num_in[b][0]
boxes_features = boxes_features_all[b, :, :N, :].reshape(
1, T * N, NFB) #1,T,N,NFB
boxes_positions = boxes_in[b, :, :N, :].reshape(T * N, 4) #T*N, 4
# GCN graph modeling
for i in range(len(self.gcn_list)):
graph_boxes_features, relation_graph = self.gcn_list[i](
boxes_features, boxes_positions)
# cat graph_boxes_features with boxes_features
boxes_features = boxes_features.reshape(1, T * N, NFB)
boxes_states = graph_boxes_features + boxes_features #1, T*N, NFG
boxes_states = self.dropout_global(boxes_states)
NFS = NFG
boxes_states = boxes_states.reshape(T, N, NFS)
# Predict actions
actn_score = self.fc_actions(boxes_states) #T,N, actn_num
# Predict activities
boxes_states_pooled, _ = torch.max(boxes_states, dim=1) #T, NFS
acty_score = self.fc_activities(boxes_states_pooled) #T, acty_num
# GSN fusion
actn_score = torch.mean(actn_score,
dim=0).reshape(N, -1) #N, actn_num
acty_score = torch.mean(acty_score,
dim=0).reshape(1, -1) #1, acty_num
actions_scores.append(actn_score)
activities_scores.append(acty_score)
actions_scores = torch.cat(actions_scores, dim=0) #ALL_N,actn_num
activities_scores = torch.cat(activities_scores, dim=0) #B,acty_num
if not self.training:
B = B // 3
actions_scores = torch.mean(actions_scores.reshape(
-1, 3, actions_scores.shape[1]),
dim=1)
activities_scores = torch.mean(activities_scores.reshape(B, 3, -1),
dim=1).reshape(B, -1)
# print(actions_scores.shape)
# print(activities_scores.shape)
return actions_scores, activities_scores
def compute_loss(self, inferences, labels, regs):
labels = torch.reshape(labels, [-1, 1])
loss = F.mse_loss(inferences, labels)
return loss + regs
def attack_single_run_targeted(self, x, y=None, use_rand_start=False):
"""
:param x: clean images
:param y: clean labels, if None we use the predicted labels
"""
if self.device is None:
self.device = x.device
self.orig_dim = list(x.shape[1:])
self.ndims = len(self.orig_dim)
x = x.detach().clone().float().to(self.device)
# assert next(self.predict.parameters()).device == x.device
y_pred = self._get_predicted_label(x)
if y is None:
y = y_pred.detach().clone().long().to(self.device)
else:
y = y.detach().clone().long().to(self.device)
pred = y_pred == y
corr_classified = pred.float().sum()
if self.verbose:
print('Clean accuracy: {:.2%}'.format(pred.float().mean()))
if pred.sum() == 0:
return x
pred = self.check_shape(pred.nonzero().squeeze())
output = self.predict(x)
la_target = output.sort(dim=-1)[1][:, -self.target_class]
startt = time.time()
# runs the attack only on correctly classified points
im2 = x[pred].detach().clone()
la2 = y[pred].detach().clone()
la_target2 = la_target[pred].detach().clone()
if len(im2.shape) == self.ndims:
im2 = im2.unsqueeze(0)
bs = im2.shape[0]
u1 = torch.arange(bs)
adv = im2.clone()
adv_c = x.clone()
res2 = 1e10 * torch.ones([bs]).to(self.device)
res_c = torch.zeros([x.shape[0]]).to(self.device)
x1 = im2.clone()
x0 = im2.clone().reshape([bs, -1])
counter_restarts = 0
while counter_restarts < 1:
if use_rand_start:
if self.norm == 'Linf':
t = 2 * torch.rand(x1.shape).to(self.device) - 1
x1 = im2 + (torch.min(
res2,
self.eps * torch.ones(res2.shape).to(self.device)
).reshape([-1, *[1] * self.ndims])) * t / (t.reshape([
t.shape[0], -1
]).abs().max(dim=1, keepdim=True)[0].reshape(
[-1, *[1] * self.ndims])) * .5
elif self.norm == 'L2':
t = torch.randn(x1.shape).to(self.device)
x1 = im2 + (torch.min(
res2,
self.eps * torch.ones(res2.shape).to(self.device)
).reshape([-1, *[1] * self.ndims])) * t / (
(t**2).view(t.shape[0], -1).sum(dim=-1).sqrt().view(
t.shape[0], *[1] * self.ndims)) * .5
elif self.norm == 'L1':
t = torch.randn(x1.shape).to(self.device)
x1 = im2 + (torch.min(
res2,
self.eps * torch.ones(res2.shape).to(self.device)
).reshape([-1, *[1] * self.ndims])) * t / (t.abs().view(
t.shape[0], -1).sum(dim=-1).view(
t.shape[0], *[1] * self.ndims)) / 2
x1 = x1.clamp(0.0, 1.0)
counter_iter = 0
while counter_iter < self.n_iter:
with torch.no_grad():
df, dg = self.get_diff_logits_grads_batch_targeted(
x1, la2, la_target2)
if self.norm == 'Linf':
dist1 = df.abs() / (1e-12 + dg.abs().view(
dg.shape[0], dg.shape[1], -1).sum(dim=-1))
elif self.norm == 'L2':
dist1 = df.abs() / (1e-12 + (dg**2).view(
dg.shape[0], dg.shape[1], -1).sum(dim=-1).sqrt())
elif self.norm == 'L1':
dist1 = df.abs() / (1e-12 + dg.abs().reshape(
[df.shape[0], df.shape[1], -1]).max(dim=2)[0])
else:
raise ValueError('norm not supported')
ind = dist1.min(dim=1)[1]
# print(ind)
dg2 = dg[u1, ind]
b = (-df[u1, ind] +
(dg2 * x1).view(x1.shape[0], -1).sum(dim=-1))
w = dg2.reshape([bs, -1])
if self.norm == 'Linf':
d3 = self.projection_linf(
torch.cat((x1.reshape([bs, -1]), x0), 0),
torch.cat((w, w), 0), torch.cat((b, b), 0))
elif self.norm == 'L2':
d3 = self.projection_l2(
torch.cat((x1.reshape([bs, -1]), x0), 0),
torch.cat((w, w), 0), torch.cat((b, b), 0))
elif self.norm == 'L1':
d3 = self.projection_l1(
torch.cat((x1.reshape([bs, -1]), x0), 0),
torch.cat((w, w), 0), torch.cat((b, b), 0))
d1 = torch.reshape(d3[:bs], x1.shape)
d2 = torch.reshape(d3[-bs:], x1.shape)
if self.norm == 'Linf':
a0 = d3.abs().max(dim=1, keepdim=True)[0] \
.view(-1, *[1] * self.ndims)
elif self.norm == 'L2':
a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt() \
.view(-1, *[1] * self.ndims)
elif self.norm == 'L1':
a0 = d3.abs().sum(dim=1, keepdim=True) \
.view(-1, *[1] * self.ndims)
a0 = torch.max(a0,
1e-8 * torch.ones(a0.shape).to(self.device))
a1 = a0[:bs]
a2 = a0[-bs:]
alpha = torch.min(
torch.max(a1 / (a1 + a2),
torch.zeros(a1.shape).to(self.device)),
self.alpha_max * torch.ones(a1.shape).to(self.device))
x1 = ((x1 + self.eta * d1) * (1 - alpha) +
(im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0)
is_adv = self._get_predicted_label(x1) != la2
if is_adv.sum() > 0:
ind_adv = is_adv.nonzero().squeeze()
ind_adv = self.check_shape(ind_adv)
if self.norm == 'Linf':
t = (x1[ind_adv] - im2[ind_adv]).reshape(
[ind_adv.shape[0], -1]).abs().max(dim=1)[0]
elif self.norm == 'L2':
t = ((x1[ind_adv] - im2[ind_adv]) ** 2) \
.view(ind_adv.shape[0], -1).sum(dim=-1).sqrt()
elif self.norm == 'L1':
t = (x1[ind_adv] - im2[ind_adv]) \
.abs().view(ind_adv.shape[0], -1).sum(dim=-1)
adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]). \
float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] \
* (t >= res2[ind_adv]).float().reshape(
[-1, *[1] * self.ndims])
res2[ind_adv] = t * (t < res2[ind_adv]).float() \
+ res2[ind_adv] * (t >= res2[ind_adv]).float()
x1[ind_adv] = im2[ind_adv] + (x1[ind_adv] -
im2[ind_adv]) * self.beta
counter_iter += 1
counter_restarts += 1
ind_succ = res2 < 1e10
if self.verbose:
print('success rate: {:.0f}/{:.0f}'.format(ind_succ.float().sum(),
corr_classified) +
' (on correctly classified points) in {:.1f} s'.format(
time.time() - startt))
res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float())
ind_succ = self.check_shape(ind_succ.nonzero().squeeze())
adv_c[pred[ind_succ]] = adv[ind_succ].clone()
return adv_c
optimizer = torch.optim.SGD([model.W, model.b], learning_rate)
for epoch in range(epochs):
model.loss(x_train, y_train).backward() # Compute loss gradients
optimizer.step() # Perform optimization by adjusting W and b
optimizer.zero_grad() # Clear gradients for next step
# Print model variables and loss
print("W = %s, b = %s, loss = %s" %
(model.W, model.b, model.loss(x_train, y_train)))
# Visualize result
fig = plt.figure("NOT operator")
plot1 = fig.add_subplot()
plt.plot(x_train.detach(),
y_train.detach(),
'o',
label='$(\\hat x^{(i)},\\hat y^{(i)})$')
plt.xlabel('x')
plt.ylabel('y')
out = torch.reshape(torch.tensor(np.linspace(0, 1, 100).tolist()), (-1, 1))
plot1.set_xticks([0, 1]) # x range from 0 to 1
plot1.set_yticks([0, 1]) # y range from 0 to 1
x, indices = torch.sort(out, 0)
# Plot sigmoid regression curve.
plt.plot(x, model.f(x).detach())
plt.show()
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
#print('Number of model parameters: {}'.format(
# len([p.data.nelement() for p in model.parameters()])))
#print(model)
###################################################Training the data########################################
for ep in range(num_epochs):
epoch_file = open(filename, 'a')
epoch_file.write('Epoch {}/{} --- \n'.format(ep + 1, num_epochs))
epoch_file.close()
for i, (text, label) in enumerate(train_loader):
#print(text[0,:10],label[0])
if torch.cuda.is_available():
text = torch.reshape(text, (batch_size, 1, 1024))
text = text.long()
text = Variable(text.cuda())
label = Variable(label.cuda())
optimizer.zero_grad()
predict = model.forward(text)
temp = 0
loss = loss_function(predict, label)
loss.backward()
optimizer.step()
k = 0.0
for i, (text, label) in enumerate(test_loader):
if torch.cuda.is_available():
text = torch.reshape(text, (batch_size, 1, 1024))
text = text.long()
def _train(self, BATCH):
obs = get_first_vector(BATCH.obs) # [T, B, S]
obs_ = get_first_vector(BATCH.obs_) # [T, B, S]
_timestep = obs.shape[0]
_batchsize = obs.shape[1]
predicted_obs_ = self._forward_dynamic_model(obs,
BATCH.action) # [T, B, S]
predicted_reward = self._reward_model(obs, BATCH.action) # [T, B, 1]
predicted_done_dist = self._done_model(obs, BATCH.action) # [T, B, 1]
_obs_loss = F.mse_loss(obs_, predicted_obs_) # todo
_reward_loss = F.mse_loss(BATCH.reward, predicted_reward)
_done_loss = -predicted_done_dist.log_prob(BATCH.done).mean()
wm_loss = _obs_loss + _reward_loss + _done_loss
self._wm_oplr.optimize(wm_loss)
obs = th.reshape(obs, (_timestep * _batchsize, -1)) # [T*B, S]
obs_ = th.reshape(obs_, (_timestep * _batchsize, -1)) # [T*B, S]
actions = th.reshape(BATCH.action,
(_timestep * _batchsize, -1)) # [T*B, A]
rewards = th.reshape(BATCH.reward,
(_timestep * _batchsize, -1)) # [T*B, 1]
dones = th.reshape(BATCH.done,
(_timestep * _batchsize, -1)) # [T*B, 1]
rollout_rewards = [rewards]
rollout_dones = [dones]
r_obs_ = obs_
_r_obs = deepcopy(BATCH.obs_)
r_done = (1. - dones)
for _ in range(self._roll_out_horizon):
r_obs = r_obs_
_r_obs.vector.vector_0 = r_obs
if self.is_continuous:
action_target = self.actor.t(_r_obs) # [T*B, A]
if self.use_target_action_noise:
r_action = self.target_noised_action(
action_target) # [T*B, A]
else:
target_logits = self.actor.t(_r_obs) # [T*B, A]
target_cate_dist = td.Categorical(logits=target_logits)
target_pi = target_cate_dist.sample() # [T*B,]
r_action = F.one_hot(target_pi, self.a_dim).float() # [T*B, A]
r_obs_ = self._forward_dynamic_model(r_obs, r_action) # [T*B, S]
r_reward = self._reward_model(r_obs, r_action) # [T*B, 1]
r_done = r_done * (1. - self._done_model(r_obs, r_action).sample()
) # [T*B, 1]
rollout_rewards.append(r_reward) # [H+1, T*B, 1]
rollout_dones.append(r_done) # [H+1, T*B, 1]
_r_obs.vector.vector_0 = obs
q = self.critic(_r_obs, actions) # [T*B, 1]
_r_obs.vector.vector_0 = r_obs_
q_target = self.critic.t(_r_obs, r_action) # [T*B, 1]
dc_r = rewards
for t in range(1, self._roll_out_horizon):
dc_r += (self.gamma**t) * (rollout_rewards[t] * rollout_dones[t])
dc_r += (self.gamma**self._roll_out_horizon) * rollout_dones[
self._roll_out_horizon] * q_target # [T*B, 1]
td_error = dc_r - q # [T*B, 1]
q_loss = td_error.square().mean() # 1
self.critic_oplr.optimize(q_loss)
# train actor
if self.is_continuous:
mu = self.actor(BATCH.obs,
begin_mask=BATCH.begin_mask) # [T, B, A]
else:
logits = self.actor(BATCH.obs,
begin_mask=BATCH.begin_mask) # [T, B, A]
logp_all = logits.log_softmax(-1) # [T, B, A]
gumbel_noise = td.Gumbel(0, 1).sample(logp_all.shape) # [T, B, A]
_pi = ((logp_all + gumbel_noise) / self.discrete_tau).softmax(
-1) # [T, B, A]
_pi_true_one_hot = F.one_hot(_pi.argmax(-1),
self.a_dim).float() # [T, B, A]
_pi_diff = (_pi_true_one_hot - _pi).detach() # [T, B, A]
mu = _pi_diff + _pi # [T, B, A]
q_actor = self.critic(BATCH.obs, mu,
begin_mask=BATCH.begin_mask) # [T, B, 1]
actor_loss = -q_actor.mean() # 1
self.actor_oplr.optimize(actor_loss)
return th.ones_like(BATCH.reward), {
'LEARNING_RATE/wm_lr': self._wm_oplr.lr,
'LEARNING_RATE/actor_lr': self.actor_oplr.lr,
'LEARNING_RATE/critic_lr': self.critic_oplr.lr,
'LOSS/wm_loss': wm_loss,
'LOSS/actor_loss': actor_loss,
'LOSS/critic_loss': q_loss,
'Statistics/q_min': q.min(),
'Statistics/q_mean': q.mean(),
'Statistics/q_max': q.max()
}
def forward(self, x):
"""Applies network layers and ops on input image(s) x.
Args:
x: input image or batch of images. Shape: [batch,3,300,300].
Return:
Depending on phase:
test:
Variable(tensor) of output class label predictions,
confidence score, and corresponding location predictions for
each object detected. Shape: [batch,topk,7]
train:
list of concat outputs from:
1: confidence layers, Shape: [batch*num_priors,num_classes]
2: localization layers, Shape: [batch,num_priors*4]
3: priorbox layers, Shape: [2,num_priors*4]
"""
size = x.size()[2:]
batch_size = x.shape[0]
sources = list()
loc = list()
conf = list()
x = self.conv_top(x)
s = self.L2Norm3_3(x)
sources.append(s)
patches = self.unfold(x)
patches = torch.cat(torch.unbind(patches, dim=2), dim=0)
patches = torch.reshape(patches, (-1, 4, 8, 8))
output_x = int((x.shape[2] - 8) / 4 + 1)
output_y = int((x.shape[3] - 8) / 4 + 1)
rnnX = self.rnn_model(patches, int(batch_size) * output_x * output_y)
x = torch.stack(torch.split(rnnX,
split_size_or_sections=int(batch_size),
dim=0),
dim=2)
x = F.fold(x, kernel_size=(1, 1), output_size=(output_x, output_y))
x = F.pad(x, (0, 1, 0, 1), mode='replicate')
for k in range(4):
x = self.mob[k](x)
s = self.L2Norm4_3(x)
sources.append(s)
for k in range(4, 8):
x = self.mob[k](x)
s = self.L2Norm5_3(x)
sources.append(s)
for k in range(8, 10):
x = self.mob[k](x)
sources.append(x)
for k in range(10, 11):
x = self.mob[k](x)
sources.append(x)
for k in range(11, 12):
x = self.mob[k](x)
sources.append(x)
# apply multibox head to source layers
loc_x = self.loc[0](sources[0])
conf_x = self.conf[0](sources[0])
loc_x = self.loc[1](loc_x)
conf_x = self.conf[1](conf_x)
max_conf, _ = torch.max(conf_x[:, 0:3, :, :], dim=1, keepdim=True)
conf_x = torch.cat((max_conf, conf_x[:, 3:, :, :]), dim=1)
loc.append(loc_x.permute(0, 2, 3, 1).contiguous())
conf.append(conf_x.permute(0, 2, 3, 1).contiguous())
for i in range(1, len(sources)):
x = sources[i]
conf.append(self.conf[i + 1](x).permute(0, 2, 3, 1).contiguous())
loc.append(self.loc[i + 1](x).permute(0, 2, 3, 1).contiguous())
features_maps = []
for i in range(len(loc)):
feat = []
feat += [loc[i].size(1), loc[i].size(2)]
features_maps += [feat]
self.priorbox = PriorBox(size, features_maps, cfg)
self.priors = self.priorbox.forward()
loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
if self.phase == 'test':
output = detect_function(
cfg,
loc.view(loc.size(0), -1, 4), # loc preds
self.softmax(conf.view(conf.size(0), -1,
self.num_classes)), # conf preds
self.priors.type(type(x.data)) # default boxes
)
else:
output = (loc.view(loc.size(0), -1,
4), conf.view(conf.size(0), -1,
self.num_classes), self.priors)
return output, loc, conf
def forward(self, input):
input = torch.reshape(
input, (input.size(0), 1, input.size(2) * input.size(3)))
self.X = input
return super().forward(input)
# Get the most probable last output index
next_word_encoded = net_out[:, -1, :]
closest_value = np.linalg.norm(
(X - next_word_encoded.to('cpu').numpy()[0]), axis=1)
closest_word = emb.index[np.argmin(closest_value)]
state += ' ' + closest_word
# Print the seed letters
#print(state, end=' ', flush=True)
#%% Generate sonnet
new_line_count = 0
tot_char_count = 0
while True:
with torch.no_grad(): # No need to track the gradients
# The new network input is the one hot encoding of the last chosen letter
net_input = ds.encode_text(encoder, state.lower(), emb)
net_input = torch.reshape(
torch.tensor(net_input).float(), (1, -1, 50))
#print(net_input.shape)
#net_input = net_input.unsqueeze(0)
# Forward pass
#net_input = net_input.to(device)
net_out, net_state = net(net_input, net_state)
# Get the most probable letter index
closest_value = np.linalg.norm(
(X - net_out[:, -1, :].to('cpu').numpy()[0]), axis=1)
closest_word = emb.index[np.argmin(closest_value)]
state += ' ' + closest_word
#print(closest_word, end=' ', flush=True)
# Count total letters
tot_char_count += 1
# Count new lines
def drop_block_2d(
x,
drop_prob: float = 0.1,
block_size: int = 7,
gamma_scale: float = 1.0,
with_noise: bool = False,
inplace: bool = False,
batchwise: bool = False,
):
"""DropBlock. See https://arxiv.org/pdf/1810.12890.pdf
DropBlock with an experimental gaussian noise option. This layer has been tested on a few training
runs with success, but needs further validation and possibly optimization for lower runtime impact.
"""
B, C, H, W = x.shape
total_size = W * H
clipped_block_size = min(block_size, min(W, H))
# seed_drop_rate, the gamma parameter
gamma = (gamma_scale * drop_prob * total_size / clipped_block_size**2 /
((W - block_size + 1) * (H - block_size + 1)))
# Forces the block to be inside the feature map.
w_i, h_i = torch.meshgrid(
torch.arange(W).to(x.device),
torch.arange(H).to(x.device))
valid_block = ((w_i >= clipped_block_size // 2) &
(w_i < W - (clipped_block_size - 1) // 2)) & (
(h_i >= clipped_block_size // 2) &
(h_i < H - (clipped_block_size - 1) // 2))
valid_block = torch.reshape(valid_block, (1, 1, H, W)).to(dtype=x.dtype)
if batchwise:
# one mask for whole batch, quite a bit faster
uniform_noise = torch.rand((1, C, H, W),
dtype=x.dtype,
device=x.device)
else:
uniform_noise = torch.rand_like(x)
block_mask = ((2 - gamma - valid_block + uniform_noise) >= 1).to(
dtype=x.dtype)
block_mask = -F.max_pool2d(
-block_mask,
kernel_size=clipped_block_size, # block_size,
stride=1,
padding=clipped_block_size // 2,
)
if with_noise:
normal_noise = (torch.randn(
(1, C, H, W), dtype=x.dtype, device=x.device)
if batchwise else torch.randn_like(x))
if inplace:
x.mul_(block_mask).add_(normal_noise * (1 - block_mask))
else:
x = x * block_mask + normal_noise * (1 - block_mask)
else:
normalize_scale = (
block_mask.numel() /
block_mask.to(dtype=torch.float32).sum().add(1e-7)).to(x.dtype)
if inplace:
x.mul_(block_mask * normalize_scale)
else:
x = x * block_mask * normalize_scale
return x
def train(self):
"""
Implement your training algorithm here
"""
for episode in range(self.episodes):
state = self.env.reset()
state = torch.reshape(tensor(state, dtype=torch.float32), [1, 84, 84, 4]).permute(0, 3, 1, 2).to(
self.device)
done = False
episode_reward = []
episode_loss = []
# save network
if episode % self.model_save_interval == 0:
save_path = self.model_save_path + '/' + self.run_name + '_' + str(episode) + '.pt'
torch.save(self.q_network.state_dict(), save_path)
print('Successfully saved: ' + save_path)
while not done:
# update target network
if self.step % self.network_update_interval == 0:
print('Updating target network')
self.target_network.load_state_dict(self.q_network.state_dict())
if self.step > len(self.replay_memory):
self.epsilon = max(self.final_epsilon, self.initial_epsilon - self.epsilon_step * self.step)
if self.epsilon > self.final_epsilon:
self.mode = 'Explore'
else:
self.mode = 'Exploit'
action, q = self.make_action(state, 0, test=False)
next_state, reward, done, _ = self.env.step(action)
next_state = torch.reshape(tensor(next_state, dtype=torch.float32), [1, 84, 84, 4]).permute(0, 3, 1,
2).to(
self.device)
self.push((state, torch.tensor([int(action)]), torch.tensor([reward], device=self.device), next_state,
torch.tensor([done], dtype=torch.float32)))
episode_reward.append(reward)
self.step += 1
state = next_state
# train network
if self.step >= self.start_to_learn and self.step % self.network_train_interval == 0:
loss = self.optimize_network()
episode_loss.append(loss)
if done:
print('Episode:', episode, ' | Steps:', self.step, ' | Eps: ', self.epsilon, ' | Reward: ',
sum(episode_reward),
' | Avg Reward: ', np.mean(self.last_n_rewards), ' | Loss: ',
np.mean(episode_loss), ' | Intrinsic Reward: ', sum(self.intrinsic_episode_reward),
' | Avg Intrinsic Reward: ', np.mean(self.last_n_intrinsic_rewards),
' | Mode: ', self.mode)
print('Episode:', episode, ' | Steps:', self.step, ' | Eps: ', self.epsilon, ' | Reward: ',
sum(episode_reward),
' | Avg Reward: ', np.mean(self.last_n_rewards), ' | Loss: ',
np.mean(episode_loss), ' | Intrinsic Reward: ', sum(self.intrinsic_episode_reward),
' | Avg Intrinsic Reward: ', np.mean(self.last_n_intrinsic_rewards),
' | Mode: ', self.mode, file=self.log_file)
self.log_summary(episode, episode_loss, episode_reward)
self.last_n_rewards.append(sum(episode_reward))
self.last_n_intrinsic_rewards.append(sum(self.intrinsic_episode_reward))
episode_reward.clear()
episode_loss.clear()
self.intrinsic_episode_reward.clear()
def forward(self, batch_data):
images_in, boxes_in = batch_data
# read config parameters
B = images_in.shape[0]
T = images_in.shape[1]
H, W = self.cfg.image_size
OH, OW = self.cfg.out_size
N = self.cfg.num_boxes
NFB = self.cfg.num_features_boxes
NFR, NFG = self.cfg.num_features_relation, self.cfg.num_features_gcn
NG = self.cfg.num_graph
D = self.cfg.emb_features
K = self.cfg.crop_size[0]
if not self.training:
B = B * 3
T = T // 3
images_in.reshape((B, T) + images_in.shape[2:])
boxes_in.reshape((B, T) + boxes_in.shape[2:])
# Reshape the input data
images_in_flat = torch.reshape(images_in,
(B * T, 3, H, W)) # B*T, 3, H, W
boxes_in_flat = torch.reshape(boxes_in, (B * T * N, 4)) # B*T*N, 4
boxes_idx = [i * torch.ones(N, dtype=torch.int) for i in range(B * T)]
boxes_idx = torch.stack(boxes_idx).to(device=boxes_in.device) # B*T, N
boxes_idx_flat = torch.reshape(boxes_idx, (B * T * N, )) # B*T*N,
# Use backbone to extract features of images_in
# Pre-precess first
images_in_flat = prep_images(images_in_flat)
outputs = self.backbone(images_in_flat)
# Build features
assert outputs[0].shape[2:4] == torch.Size([OH, OW])
features_multiscale = []
for features in outputs:
if features.shape[2:4] != torch.Size([OH, OW]):
features = F.interpolate(features,
size=(OH, OW),
mode='bilinear',
align_corners=True)
features_multiscale.append(features)
features_multiscale = torch.cat(features_multiscale,
dim=1) # B*T, D, OH, OW
# RoI Align
boxes_in_flat.requires_grad = False
boxes_idx_flat.requires_grad = False
boxes_features = self.roi_align(features_multiscale, boxes_in_flat,
boxes_idx_flat) # B*T*N, D, K, K,
boxes_features = boxes_features.reshape(B, T, N, -1) # B,T,N, D*K*K
# Embedding
boxes_features = self.fc_emb_1(boxes_features) # B,T,N, NFB
boxes_features = self.nl_emb_1(boxes_features)
boxes_features = F.relu(boxes_features)
# GCN
graph_boxes_features = boxes_features.reshape(B, T * N, NFG)
# visual_info=[]
for i in range(len(self.gcn_list)):
graph_boxes_features, relation_graph = self.gcn_list[i](
graph_boxes_features, boxes_in_flat)
# visual_info.append(relation_graph.reshape(B,T,N,N))
# fuse graph_boxes_features with boxes_features
graph_boxes_features = graph_boxes_features.reshape(B, T, N, NFG)
boxes_features = boxes_features.reshape(B, T, N, NFB)
# boxes_states= torch.cat( [graph_boxes_features,boxes_features],dim=3) #B, T, N, NFG+NFB
boxes_states = graph_boxes_features + boxes_features
boxes_states = self.dropout_global(boxes_states)
NFS = NFG
# Predict actions
boxes_states_flat = boxes_states.reshape(-1, NFS) # B*T*N, NFS
actions_scores = self.fc_actions(boxes_states_flat) # B*T*N, actn_num
# Predict activities
boxes_states_pooled, _ = torch.max(boxes_states, dim=2)
boxes_states_pooled_flat = boxes_states_pooled.reshape(-1, NFS)
activities_scores = self.fc_activities(
boxes_states_pooled_flat) # B*T, acty_num
# Temporal fusion
actions_scores = actions_scores.reshape(B, T, N, -1)
actions_scores = torch.mean(actions_scores, dim=1).reshape(B * N, -1)
activities_scores = activities_scores.reshape(B, T, -1)
activities_scores = torch.mean(activities_scores, dim=1).reshape(B, -1)
if not self.training:
B = B // 3
actions_scores = torch.mean(actions_scores.reshape(B, 3, N, -1),
dim=1).reshape(B * N, -1)
activities_scores = torch.mean(activities_scores.reshape(B, 3, -1),
dim=1).reshape(B, -1)
return actions_scores, activities_scores
seed = 1234
#####
### Shuffling the list to randomize the data
random.shuffle(training_files)
######
N_batches = 10
for epoch in range(N_epochs):
loss_sum = 0
err_sum = 0
acc_sum = 0
for i in range(N_batches):
input_data, labels = create_batches_rnd(batch_size, training_files,
N_train_files, window_len, 0.2)
inputs = torch.reshape(input_data,
(batch_size, 1, input_shape[0], input_shape[1]))
inputs = inputs.to(device)
labels = labels.to(device)
output = model(inputs)
loss = criterion(output, labels.long())
prediction = torch.max(output, dim=1)[1]
err = torch.mean((prediction != labels.long()).float())
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
#print(loss.item())
loss_sum = loss_sum + loss.detach()
err_sum = err_sum + err.detach()
acc_sum = acc_sum + torch.mean((prediction == labels.long()).float())
loss_tot = loss_sum / N_batches
def unit_test_regressR(self):
print('Unit test regressR')
# Only regress G0, the rest can be solved numerically.
real_thetas = torch.from_numpy((np.random.rand(32, pose_size) - 0.5) * 1.5)\
.type(torch.float64).to(self.device)
'''
Fix global rotation, change local rotation
'''
# Test if change R[23] affects joint 23, and the shape of finger tips?
# Check passed, change leaf rotation does not affect leaf joints, but do affect the body shapes
# Consider adding another control point at head to determine head rotation.
undefined = [
30, 31, 32, 33, 34, 35, 45, 46, 47, 66, 67, 68, 69, 70, 71
]
index = torch.zeros(72, dtype=torch.int, device=self.device)
index[undefined] = 1
for j in range(1, 32):
real_thetas[j] = torch.where(index == 0, real_thetas[0],
real_thetas[j])
#print('thetas:', real_thetas)
betas = torch.from_numpy(np.zeros((32, beta_size))) \
.type(torch.float64).to(self.device)
trans = torch.from_numpy(np.zeros(
(32, 3))).type(torch.float64).to(self.device)
meshes, joints = self.forward(betas, real_thetas, trans)
#reg_G = self.regressG(joints)
v_shaped = torch.tensordot(betas, self.shapedirs,
dims=([1], [2])) + self.v_template
J = torch.matmul(self.J_regressor, v_shaped)
G, R_cube_big = self.theta2G(real_thetas,
J) # pre-calculate G terms for skinning
# print(G[0,0])
# R00 = G[0,0,:3,:3]
# print('j0:',self.J0[0])
# j0_ = torch.matmul(
# torch.eye(3, dtype=torch.float64, device =self.device) - R00,
# self.J0[0]
# )
# print('(I-R_0)J0:',j0_)
# What if we directly apply G on J?
J_1 = torch.cat((J, torch.ones(
(32, J.shape[1], 1), dtype=torch.float64).to(self.device)),
dim=2).reshape(32, -1, 4, 1)
fake_joints = torch.matmul(G, J_1)
fake_joints = torch.reshape(fake_joints, (32, -1, 4))[:, :, :3]
#print('G_0J0:',fake_joints[0,0])
#print('real joint_0:',joints[0,0])
# Test if directly regress joints from G works...
# 20190308: Good approximation, visually undiscernable.
for i in range(32):
model.write_obj(meshes[i].detach().cpu().numpy(),
'./joint_test_0308/real_{}.obj'.format(i))
np.savetxt('./joint_test_0308/real_{}.xyz'.format(i),
joints[i].detach().cpu().numpy(),
delimiter=' ')
np.savetxt('./joint_test_0308/fake_{}.xyz'.format(i),
fake_joints[i].detach().cpu().numpy(),
delimiter=' ')
def main():
TRAIN_IMG_DIR = "data/images"
VAL_IMG_DIR = "data/images"
LABEL_DIR = "data/labels"
BATCH_SIZE = 16
NUM_WORKERS = 2
PIN_MEMORY = True
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
LEARNING_RATE = 2e-4
WEIGHT_DECAY = 0.0
EPOCHS = 100
TRAIN = False
LOAD_MODEL_FILE = "epoch_trained.pth.tar"
S = 7
C = 20
B = 2
print(DEVICE)
class Compose(object):
def __init__(self, transforms):
self.transforms = transforms
def __call__(self, image, bboxes):
for t in self.transforms:
image = t(image)
return image, bboxes
# Types of preprocessing transforms we want to apply
convert_transform = transforms.ToTensor()
resize_transform = transforms.Resize((448, 448))
# normalize_transform = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
transform = Compose([convert_transform, resize_transform])
train_dataset = VOCDataset("data/train.csv",
S=S,
C=C,
B=B,
transform=transform,
img_dir=VAL_IMG_DIR,
label_dir=LABEL_DIR)
train_loader = DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=True)
val_dataset = VOCDataset("data/train.csv",
S=S,
C=C,
B=B,
transform=transform,
img_dir=VAL_IMG_DIR,
label_dir=LABEL_DIR)
val_loader = DataLoader(dataset=val_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=True)
model = YoloV1(S=S, B=B, C=C).to(DEVICE)
for child in model.resnet18.parameters():
child.requires_grad = True
model.resnet18.fc.requires_grad = True
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=LEARNING_RATE,
weight_decay=WEIGHT_DECAY)
yolo_loss = YoloLoss(S=S, C=C, B=B)
writer = SummaryWriter()
# TRAINING
if TRAIN:
mean_loss_list = []
for epoch in tqdm(range(EPOCHS)):
mean_loss = []
for batch_idx, (x, y) in enumerate(train_loader):
x, y = x.to(DEVICE), y.to(DEVICE)
y_p = model(x)
# y = torch.flatten(y, start_dim=1, end_dim=3)
y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B))
loss = yolo_loss(y_p, y)
mean_loss.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Calculate average loss
mean_loss = sum(mean_loss) / len(mean_loss)
writer.add_scalar("Average Loss: ", mean_loss, epoch)
print("Epoch:", epoch)
print(f"Mean loss was {mean_loss}")
mean_loss_list.append(mean_loss)
# Save the model
if True:
checkpoint = {
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
}
print("=> Saving checkpoint")
torch.save(checkpoint, LOAD_MODEL_FILE)
time.sleep(10)
writer.close()
plt.plot(mean_loss_list)
plt.show()
# VALIDATION
else:
print("=> Loading checkpoint")
model.load_state_dict(torch.load(LOAD_MODEL_FILE)["state_dict"])
optimizer.load_state_dict(torch.load(LOAD_MODEL_FILE)["optimizer"])
model.train() # Sets model to training mode
model.eval() # Sets model to evaluation mode
with torch.no_grad(): # Use with evaluation mode
for epoch in tqdm(range(1)):
for i, (x, y) in enumerate(val_loader):
x, y = x.to(DEVICE), y.to(DEVICE)
y_p = model(x)
x = x.to('cpu')
y_p = y_p.to('cpu')
y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B))
print("Rendering original labels (y to x)")
render_batch(x, y)
print("Rendering predicted labels (y_p to x)")
render_batch(x, y_p)
def __class_predictor(self, feature_maps):
convoutput = F.conv2d(feature_maps, self.class_predictor_weights)
new_shape = convoutput.shape[0:1] + torch.Size(
[self.num_classes, self.nboxes_per_pixel]) + convoutput.shape[2:4]
# dont take softmax here, take logsoftmax in the loss
return torch.reshape(convoutput, new_shape)
def forward(self, betas, thetas, trans, gR=None):
"""
Construct a compute graph that takes in parameters and outputs a tensor as
model vertices. Face indices are also returned as a numpy ndarray.
20190128: Add batch support.
20190322: Extending forward compatiability with SMPLModelv3
Usage:
---------
meshes, joints = forward(betas, thetas, trans): normal SMPL
meshes, joints = forward(betas, thetas, trans, gR=gR):
calling from SMPLModelv3, using gR to cache G terms, ignoring thetas
Parameters:
---------
thetas: an [N, 24 * 3] tensor indicating child joint rotation
relative to parent joint. For root joint it's global orientation.
Represented in a axis-angle format.
betas: Parameter for model shape. A tensor of shape [N, 10] as coefficients of
PCA components. Only 10 components were released by SMPL author.
trans: Global translation tensor of shape [N, 3].
G, R_cube_big: (Added on 0322) Fix compatible issue when calling from v3 objects
when calling this mode, theta must be set as None
Return:
------
A 3-D tensor of [N * 6890 * 3] for vertices,
and the corresponding [N * 24 * 3] joint positions.
"""
batch_num = betas.shape[0]
v_shaped = torch.tensordot(betas, self.shapedirs,
dims=([1], [2])) + self.v_template
J = torch.matmul(self.J_regressor, v_shaped)
if gR is not None:
G, R_cube_big = self.gR2G(gR, J)
elif thetas is not None:
G, R_cube_big = self.theta2G(
thetas, J) # pre-calculate G terms for skinning
else:
raise (RuntimeError(
'Either thetas or gR should be specified, but detected two Nonetypes'
))
# (1) Pose shape blending (SMPL formula(9))
if self.simplify:
v_posed = v_shaped
else:
R_cube = R_cube_big[:, 1:, :, :]
I_cube = (torch.eye(3, dtype=self.data_type).unsqueeze(dim=0) + \
torch.zeros((batch_num, R_cube.shape[1], 3, 3), dtype=self.data_type)).to(self.device)
lrotmin = (R_cube - I_cube).reshape(batch_num, -1)
v_posed = v_shaped + torch.tensordot(
lrotmin, self.posedirs, dims=([1], [2]))
# (2) Skinning (W)
T = torch.tensordot(G, self.weights,
dims=([1], [1])).permute(0, 3, 1, 2)
rest_shape_h = torch.cat((v_posed,
torch.ones(
(batch_num, v_posed.shape[1], 1),
dtype=self.data_type).to(self.device)),
dim=2)
v = torch.matmul(T, torch.reshape(rest_shape_h, (batch_num, -1, 4, 1)))
v = torch.reshape(v, (batch_num, -1, 4))[:, :, :3]
result = v + torch.reshape(trans, (batch_num, 1, 3))
# estimate 3D joint locations
#joints = torch.tensordot(result, self.joint_regressor, dims=([1], [0])).transpose(1, 2)
joints = torch.tensordot(result,
self.J_regressor.transpose(0, 1),
dims=([1], [0])).transpose(1, 2)
return result, joints
def prepare(feat):
B, n_ch, n_voxels = feat.size()
n_ch_1 = n_ch + 1
conv_tensor = np.tril(np.ones((n_ch + 1, n_ch + 1)), -1).T
conv_tensor += np.diag([-i for i in range(n_ch + 1)])
conv_tensor = conv_tensor[:, 1:]
conv_tensor = np.matmul(
conv_tensor,
np.sqrt(np.diag([1 / (d * (d + 1)) for d in range(1, n_ch + 1)])))
inv_std_dev = np.sqrt(2 / 3.) * (n_ch + 1)
conv_tensor *= inv_std_dev
conv_tensor = conv_tensor[:, :, np.newaxis]
feat = F.conv1d(feat, torch.FloatTensor(conv_tensor).cuda())
feat_v = torch.round(feat / (n_ch + 1))
rem0 = feat_v * (n_ch + 1)
index = torch.argsort(feat - rem0, dim=1, descending=True)
rank = torch.argsort(index, dim=1, descending=False)
rank = rank + torch.sum(feat_v, 1).unsqueeze(1).type(
torch.cuda.LongTensor)
add_minus = (rank < 0).type(
torch.cuda.LongTensor) - (rank > n_ch).type(torch.cuda.LongTensor)
add_minus *= (n_ch + 1)
rank = rank + add_minus
rem0 = rem0 + add_minus.type(torch.cuda.FloatTensor)
y = (feat - rem0) / (n_ch + 1)
v_sorted = torch.sort(y, dim=1, descending=False)[0]
barycentric = v_sorted - torch.cat(
[v_sorted[:, -1:] - 1., v_sorted[:, :-1]], 1)
canonical = torch.cuda.FloatTensor([[i] * ((n_ch + 1) - i) +
[i - (n_ch + 1)] * i
for i in range(n_ch + 1)])
def _simple_hash(key):
key = key.type(torch.cuda.DoubleTensor)
hash_vector = np.floor(
np.power(np.iinfo(np.int64).max, 1. / (n_ch + 2)))
hash_vector = torch.pow(hash_vector, torch.arange(1, n_ch + 1))
hash_vector = hash_vector.type(torch.DoubleTensor).unsqueeze(0)
hash_vector = hash_vector.cuda()
if len(key.size()) == 3:
hash_vector = hash_vector.unsqueeze(2)
return torch.sum(
key * hash_vector.repeat(key.size(0), 1, key.size(-1)), 1)
if len(key.size()) == 2:
return torch.sum(key * hash_vector.repeat(key.size(0), 1), 1)
dic_hash_lattice = HashTable(n_ch, torch.cuda.DoubleTensor, 2**30)
loc = [None] * (n_ch + 1)
loc_hash = [None] * (n_ch + 1)
for scit in range(n_ch + 1):
loc[scit] = torch.gather(
canonical[scit:scit + 1].unsqueeze(2).repeat(
rank.size(0), 1, rank.size(2)), 1, rank[:, :-1])
loc[scit] += rem0[:, :-1]
loc[scit] = loc[scit]
loc_hash[scit] = _simple_hash(loc[scit])
loc[scit] = torch.reshape(loc[scit].permute(0, 2, 1), (-1, n_ch))
dic_hash_lattice.add_values(loc_hash[scit].view(-1), loc[scit])
dic_hash_lattice.filter_values()
fused_loc = dic_hash_lattice.export_values()
dic_hash_lattice.update_rank()
indices = [None] * n_ch_1
blur_neighbours1 = [None] * n_ch_1
blur_neighbours2 = [None] * n_ch_1
default = torch.tensor(0).type(torch.LongTensor).cuda()
for dit in range(n_ch_1):
offset = [n_ch if i == dit else -1 for i in range(n_ch)]
offset = torch.cuda.FloatTensor(offset)
blur_neighbours1[dit] = dic_hash_lattice.get_rank(
_simple_hash(fused_loc + offset).view(-1))[:, 0]
blur_neighbours2[dit] = dic_hash_lattice.get_rank(
_simple_hash(fused_loc - offset).view(-1))[:, 0]
indices[dit] = dic_hash_lattice.get_rank(
loc_hash[dit].view(-1)).view(B, n_voxels)
return rank, barycentric, blur_neighbours1, blur_neighbours2, indices
def forward(self,
inputs,
input_frames=20,
future_frames=1,
output_frames=1,
channels=1,
teacher_forcing=False,
scheduled_sampling_ratio=0):
batch_size, input_frames, height = inputs.size()
output_frames = input_frames
inputs = torch.reshape(inputs,
(batch_size, input_frames, channels, height))
"""
Computation of Convolutional LSTM network.
Arguments:
----------
inputs: a 5-th order tensor of size [batch_size, input_frames, input_channels, height, width]
Input tensor (video) to the deep Conv-LSTM network.
input_frames: int
The number of input frames to the model.
future_frames: int
The number of future frames predicted by the model.
output_frames: int
The number of output frames returned by the model.
teacher_forcing: bool
Whether the model is trained in teacher_forcing mode.
Note 1: In test mode, teacher_forcing should be set as False.
Note 2: If teacher_forcing mode is on, # of frames in inputs = total_steps
If teacher_forcing mode is off, # of frames in inputs = input_frames
scheduled_sampling_ratio: float between [0, 1]
The ratio of ground-truth frames used in teacher_forcing mode.
default: 0 (i.e. no teacher forcing effectively)
Returns:
--------
outputs: a 5-th order tensor of size [batch_size, output_frames, hidden_channels, height, width]
Output frames of the convolutional-LSTM module.
"""
# compute the teacher forcing mask
if teacher_forcing and scheduled_sampling_ratio > 1e-6:
# generate the teacher_forcing mask (4-th order)
teacher_forcing_mask = torch.bernoulli(
scheduled_sampling_ratio * torch.ones(inputs.size(0),
future_frames - 1,
1,
1,
1,
device=inputs.device))
else: # if not teacher_forcing or scheduled_sampling_ratio < 1e-6:
teacher_forcing = False
# the number of time steps in the computational graph
total_steps = input_frames + future_frames - 1
outputs = [None] * total_steps
for t in range(total_steps):
# input_: 4-th order tensor of size [batch_size, input_channels, height, width]
if t < input_frames:
input_ = inputs[:, t]
elif not teacher_forcing:
input_ = outputs[t - 1]
else: # if t >= input_frames and teacher_forcing:
mask = teacher_forcing_mask[:, t - input_frames]
input_ = inputs[:, t] * mask + outputs[t - 1] * (1 - mask)
first_step = (t == 0)
queue = [] # previous outputs for skip connection
for b in range(self.num_blocks):
for l in range(self.layers_per_block[b]):
lid = "b{}l{}".format(b, l) # layer ID
input_ = self.layers[lid](input_, first_step=first_step)
queue.append(input_)
if b >= self.skip_stride:
input_ = torch.cat([input_, queue.pop(0)],
dim=1) # concat over the channels
# map the hidden states to predictive frames (with optional sigmoid function)
outputs[t] = self.layers["output"](input_)
if self.output_sigmoid:
outputs[t] = torch.sigmoid(outputs[t])
# return the last output_frames of the outputs
outputs = outputs[-output_frames:]
# 5-th order tensor of size [batch_size, output_frames, channels, height, width]
outputs = torch.stack([outputs[t] for t in range(output_frames)],
dim=1)
outputs = self.final_layer(outputs)
return outputs[:, :, :, 0]
def eval_conditional_density(
density: Any,
condition: Tensor,
limits: Tensor,
dim1: int,
dim2: int,
resolution: int = 50,
eps_margins1: Union[Tensor, float] = 1e-32,
eps_margins2: Union[Tensor, float] = 1e-32,
) -> Tensor:
r"""
Return the unnormalized conditional along `dim1, dim2` given parameters `condition`.
We compute the unnormalized conditional by evaluating the joint distribution:
$p(x1 | x2) = p(x1, x2) / p(x2) \propto p(x1, x2)$
Args:
density: Probability density function with `.log_prob()` method.
condition: Parameter set that all dimensions other than dim1 and dim2 will be
fixed to. Should be of shape (1, dim_theta), i.e. it could e.g. be
a sample from the posterior distribution. The entries at `dim1` and `dim2`
will be ignored.
limits: Bounds within which to evaluate the density. Shape (dim_theta, 2).
dim1: First dimension along which to evaluate the conditional.
dim2: Second dimension along which to evaluate the conditional.
resolution: Resolution of the grid along which the conditional density is
evaluated.
eps_margins1: We will evaluate the posterior along `dim1` at
`limits[0]+eps_margins` until `limits[1]-eps_margins`. This avoids
evaluations potentially exactly at the prior bounds.
eps_margins2: We will evaluate the posterior along `dim2` at
`limits[0]+eps_margins` until `limits[1]-eps_margins`. This avoids
evaluations potentially exactly at the prior bounds.
Returns: Conditional probabilities. If `dim1 == dim2`, this will have shape
(resolution). If `dim1 != dim2`, it will have shape (resolution, resolution).
"""
condition = ensure_theta_batched(condition)
theta_grid_dim1 = torch.linspace(
float(limits[dim1, 0] + eps_margins1),
float(limits[dim1, 1] - eps_margins1),
resolution,
)
theta_grid_dim2 = torch.linspace(
float(limits[dim2, 0] + eps_margins2),
float(limits[dim2, 1] - eps_margins2),
resolution,
)
if dim1 == dim2:
repeated_condition = condition.repeat(resolution, 1)
repeated_condition[:, dim1] = theta_grid_dim1
log_probs_on_grid = density.log_prob(repeated_condition)
else:
repeated_condition = condition.repeat(resolution**2, 1)
repeated_condition[:, dim1] = theta_grid_dim1.repeat(resolution)
repeated_condition[:, dim2] = torch.repeat_interleave(
theta_grid_dim2, resolution)
log_probs_on_grid = density.log_prob(repeated_condition)
log_probs_on_grid = torch.reshape(log_probs_on_grid,
(resolution, resolution))
# Subtract maximum for numerical stability.
return torch.exp(log_probs_on_grid - torch.max(log_probs_on_grid))
transition = agent.step()
buffer.add_to_buffer(transition)
#Only start training after 100 transitions in the buffer
mini_batch = buffer.sample_minbatch()
loss = dqn.train_q_network(mini_batch)
losses.append(loss) #append final loss for that episode
training_iteration += 1
print("Iteration:", training_iteration)
iterations.append(training_iteration)
# GET FINAL Q VALUES FOR EACH STATE (EXTRACT FINAL LAYER FROM NETWORK, WITH EXISTING WEIGHTS I THINK?)
#PROBABLY THIS RESHAPE ISN'T CORRECT, BUT SOMEHOW NEED TO WORK OUT HOW TO CORRECTLY RESHAPE FROM 4X100 TO 10X10X4.
q_values = torch.reshape((dqn.q_network.output_layer.weight.data).t(),
(10, 10, 4)).numpy()
print(q_values.shape)
# Draw final Q pattern
visualiser = QValueVisualiser(
environment=environment,
magnification=500) #not sure if this is working correctly or not???
# Draw the image
visualiser.draw_q_values(
q_values
) #expecting q_values = np.random.uniform(0, 1, [10, 10, 4])
#Draw the ideal path (the greedy policy) found from training.
#Find the greedy policy: Given a starting state, what is the max Q action, then move to that state and take next max Q to move there until end of episode.
dqn.q_network.eval() #put into eval mode so it doesn't keep training.
agent.reset()
transitions = []
def forward(self, x):
x = x.to(device)
weights = self.hyper_net(x)
weights = torch.reshape(weights, [self.output_size, -1])
return weights
from options.train_options import TrainOptions
from utils.timer import Timer
import os
from data import CreateSrcDataLoader
from data import CreateTrgDataLoader
from model import CreateModel
#import tensorboardX
import torch.backends.cudnn as cudnn
import torch
from torch.autograd import Variable
from utils import FDA_source_to_target
import scipy.io as sio
IMG_MEAN = np.array((104.00698793, 116.66876762, 122.67891434),
dtype=np.float32)
IMG_MEAN = torch.reshape(torch.from_numpy(IMG_MEAN), (1, 3, 1, 1))
CS_weights = np.array((1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0),
dtype=np.float32)
CS_weights = torch.from_numpy(CS_weights)
def main():
opt = TrainOptions()
args = opt.initialize()
os.environ["CUDA_VISIBLE_DEVICES"] = args.GPU
_t = {'iter time': Timer()}
model_name = args.source + '_to_' + args.target
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
def computeSquareEuclideanCostMatrix(self, Xs_batch, Xt_batch):
return torch.reshape(torch.sum(torch.mul(Xs_batch, Xs_batch), dim=1), (-1, 1)) + torch.reshape(torch.sum(torch.mul(Xt_batch, Xt_batch), dim=1), (1, -1)) \
- 2. * torch.matmul(Xs_batch, torch.transpose(Xt_batch, 0,1))
def forward(self, z):
z = self.linear(z)
z = torch.reshape(z, (-1, 1, int(self.outsize/8), int(self.outsize/8)))
loc_img = self.net(z)
return loc_img
def reshape(input, shape):
return th.reshape(input, shape)
def create_dense_flows(flattened_flows, batch_size, image_height, image_width):
# possibly .view
return torch.reshape(flattened_flows,
[batch_size, image_height, image_width, 2])
