def barycentric_model_batch_loss(self, u_batch, v_batch, Xs_batch, Xt_batch, fXs_batch): C_batch = self.computeSquareEuclideanCostMatrix(Xs_batch, Xt_batch) if self.reg_type == 'entropy': H = torch.exp((torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch) / self.reg_val) elif self.reg_type == 'l2': H = (1./(2.*self.reg_val))*torch.max(torch.zeros(1, dtype=self.dtype, device=self.device), (torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch)) d_batch = self.computeSquareEuclideanCostMatrix(fXs_batch, Xt_batch) return torch.sum(torch.mul(d_batch, H))
def dual_OT_batch_loss(self, batch_size, u_batch, v_batch, Xs_batch, Xt_batch): C_batch = self.computeSquareEuclideanCostMatrix(Xs_batch=Xs_batch, Xt_batch=Xt_batch) if self.reg_type == 'entropy': loss_batch = torch.sum(u_batch)*batch_size + torch.sum(v_batch)*batch_size \ - self.reg_val*torch.sum(torch.exp((torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch) / self.reg_val)) elif self.reg_type == 'l2': tmp = torch.max(torch.zeros(1, dtype=self.dtype, device=self.device), (torch.reshape(u_batch, (-1, 1)) + torch.reshape(v_batch, (1, -1)) - C_batch)) loss_batch = torch.sum(u_batch)*batch_size + torch.sum(v_batch)*batch_size \ - (1./(4.*self.reg_val))*torch.sum(torch.mul(tmp, tmp)) return -loss_batch
def forward(self, X, X_mask): #X_mask not used #X: [m, V, Tx] m = batch size, V = vocabulary size, Tx = char count (max 1014) #print(X.size(), type(X)) m = X.size()[0] V = X.size()[1] Tx = X.size()[2] #conv layer for module in self.convlayers: #print(X.size(), type(X.data)) if 'Conv1d' in str(type(module)): X = X.unsqueeze(1) X = module(X) if 'Conv1d' in str(type(module)): X = X.squeeze() #X: [m, num_filters, 14] #print(X.size(), type(X.data)) assert X.size() == torch.Size([m, self.num_filters, 14]) #linear X = torch.reshape(X, (m, 14*self.num_filters)) assert X.size() == torch.Size([m, self.num_filters*14]) X = self.linear(X) #X: [m, 1] #print(X.size(), type(X)) if self.num_classes == 2: assert X.size() == torch.Size([m, 1]) else: assert X.size() == torch.Size([m, self.num_classes]) if self.num_classes == 2: X = torch.squeeze(X) X = self.sigmoid(X) #X: [m] #print(X.size(), type(X)) assert X.size() == torch.Size([m]) return X else: return F.softmax(X)
def preprocessing_fn_torch(batch, consecutive_frames=16, scale_first=True, align_corners=False): """ inputs - batch of videos each with shape (frames, height, width, channel) outputs - batch of videos each with shape (n_stack, channel, stack_frames, new_height, new_width) frames = n_stack * stack_frames (after padding) new_height = new_width = 112 consecutive_frames - number of consecutive frames (stack_frames) After resizing, a center crop is performed to make the image square This is a differentiable alternative to MARS' PIL-based preprocessing. There are some """ if not isinstance(batch, torch.Tensor): logger.warning(f"batch {type(batch)} is not a torch.Tensor. Casting") batch = torch.from_numpy(batch).to(DEVICE) # raise ValueError(f"batch {type(batch)} is not a torch.Tensor") if batch.dtype != torch.float32: raise ValueError(f"batch {batch.dtype} should be torch.float32") if batch.shape[0] != 1: raise ValueError(f"Batch size {batch.shape[0]} != 1") video = batch[0] if video.ndim != 4: raise ValueError( f"video dims {video.ndim} != 4 (frames, height, width, channel)") if video.shape[0] < 1: raise ValueError("video must have at least one frame") if tuple(video.shape[1:]) != (240, 320, 3): raise ValueError( f"frame shape {tuple(video.shape[1:])} != (240, 320, 3)") if video.max() > 1.0 or video.min() < 0.0: raise ValueError("input should be float32 in [0, 1] range") if not isinstance(consecutive_frames, int): raise ValueError( f"consecutive_frames {consecutive_frames} must be an int") if consecutive_frames < 1: raise ValueError( f"consecutive_frames {consecutive_frames} must be positive") # Select a integer multiple of consecutive frames while len(video) < consecutive_frames: # cyclic pad if insufficient for a single stack video = torch.cat([video, video[:consecutive_frames - len(video)]]) if len(video) % consecutive_frames != 0: # cut trailing frames video = video[:len(video) - (len(video) % consecutive_frames)] if scale_first: # Attempts to directly follow MARS approach # (frames, height, width, channel) to (frames, channel, height, width) video = video.permute(0, 3, 1, 2) sample_width, sample_height = 149, 112 video = torch.nn.functional.interpolate( video, size=(sample_height, sample_width), mode="bilinear", align_corners=align_corners, ) crop_left = 18 # round((149 - 112)/2.0) video = video[:, :, :, crop_left:crop_left + sample_height] else: # More efficient, but not MARS approach # Center crop sample_size = 112 upsample, downsample = 7, 15 assert video.shape[1] * upsample / downsample == sample_size crop_width = 40 assert crop_width == (video.shape[2] - video.shape[1]) / 2 assert video.shape[1] + 2 * crop_width == video.shape[2] video = video[:, :, crop_width:video.shape[2] - crop_width, :] assert video.shape[1] == video.shape[2] == 240 # Downsample to (112, 112) frame size # (frames, height, width, channel) to (frames, channel, height, width) video = video.permute(0, 3, 1, 2) video = torch.nn.functional.interpolate( video, size=(sample_size, sample_size), mode="bilinear", align_corners=align_corners, ) if video.max() > 1.0: raise ValueError("Video exceeded max after interpolation") if video.min() < 0.0: raise ValueError("Video under min after interpolation") # reshape into stacks of frames video = torch.reshape(video, (-1, consecutive_frames) + video.shape[1:]) # transpose to (stacks, channel, stack_frames, height, width) video = video.permute(0, 2, 1, 3, 4) # video = torch.transpose(video, axes=(0, 4, 1, 2, 3)) # normalize before changing channel position? video = torch.transpose(video, 1, 4) video = ( (video * 255) - torch.from_numpy(MEAN).to(DEVICE)) / torch.from_numpy(STD).to(DEVICE) video = torch.transpose(video, 4, 1) return video
def pytorch_compute_single(im, target_interface, GPU_mode=1): input_pytorch_value = None output_pytorch_value = None input_pytorch_cpu_value = None output_pytorch_cpu_value = None pytorch_shape = [1] for shape_element in im.shape: pytorch_shape.append(shape_element) input_pytorch = torch.reshape(torch.from_numpy(im), pytorch_shape) if GPU_mode != 0: input_pytorch_cpu_value = input_pytorch.numpy() if target_interface == 'conv1': weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1))) output_pytorch_cpu = torch.from_numpy( F.conv2d(torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2))), weights_torch, padding=0, stride=4).numpy().transpose((0, 2, 3, 1))) elif target_interface == 'conv2': x_torch = torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2)).astype(np.float64)) weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1))) stride = 4 if x_torch.numpy().shape[2] % stride == 0: pad = max(weights_torch.numpy().shape[2] - stride, 0) else: pad = max(weights_torch.numpy().shape[2] - (x_torch.numpy().shape[2] % stride), 0) if pad % 2 == 0: pad_val = pad // 2 padding = (pad_val, pad_val, pad_val, pad_val) else: pad_val_start = pad // 2 pad_val_end = pad - pad_val_start padding = (pad_val_start, pad_val_end, pad_val_start, pad_val_end) x_torch = F.pad(x_torch, padding, "constant", 0) output_pytorch_cpu = torch.from_numpy( F.conv2d(x_torch, weights_torch, padding=0, stride=stride).numpy().transpose((0, 2, 3, 1))) elif target_interface == 'pool1': output_pytorch_cpu = torch.from_numpy(np.rollaxis( F.max_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_cpu_value, 3, 1)), kernel_size=(2, 2), stride=(2, 2)).numpy(), 1, 4)) elif target_interface == 'pool2': output_pytorch_cpu = torch.from_numpy(np.rollaxis( F.avg_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_cpu_value, 3, 1)), kernel_size=(2, 2), stride=(2, 2)).numpy(), 1, 4)) elif target_interface == 'relu1': output_pytorch_cpu = F.relu(input_pytorch) elif target_interface == 'dense1': output_pytorch_cpu = None elif target_interface == 'sigmoid1': output_pytorch_cpu = F.sigmoid(input_pytorch) elif target_interface == 'tanh1': output_pytorch_cpu = F.tanh(input_pytorch) else: output_pytorch_cpu = None output_pytorch_cpu_value = output_pytorch_cpu.numpy() if GPU_mode != 2: input_pytorch_value = input_pytorch.to("cuda").to("cpu").numpy() if target_interface == 'conv1': weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1))) output_pytorch = torch.from_numpy( F.conv2d(torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2))).to("cuda"), weights_torch.to("cuda"), padding=0, stride=4).to("cpu").numpy().transpose((0, 2, 3, 1))).to( "cuda") elif target_interface == 'conv2': x_torch = torch.from_numpy(input_pytorch.numpy().transpose((0, 3, 1, 2)).astype(np.float64)) weights_torch = torch.from_numpy(np.full((11, 11, 3, 1), 1, np.float64).transpose((3, 2, 0, 1))) stride = 4 if x_torch.numpy().shape[2] % stride == 0: pad = max(weights_torch.numpy().shape[2] - stride, 0) else: pad = max(weights_torch.numpy().shape[2] - (x_torch.numpy().shape[2] % stride), 0) if pad % 2 == 0: pad_val = pad // 2 padding = (pad_val, pad_val, pad_val, pad_val) else: pad_val_start = pad // 2 pad_val_end = pad - pad_val_start padding = (pad_val_start, pad_val_end, pad_val_start, pad_val_end) x_torch = F.pad(x_torch, padding, "constant", 0) output_pytorch = torch.from_numpy( F.conv2d(x_torch.to("cuda"), weights_torch.to("cuda"), padding=0, stride=4).to("cpu").numpy().transpose( (0, 2, 3, 1))).to("cuda") elif target_interface == 'pool1': output_pytorch = torch.from_numpy(np.rollaxis( F.max_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_value, 3, 1)).to("cuda"), kernel_size=(2, 2), stride=(2, 2)).to("cpu").numpy(), 1, 4)).to("cuda") elif target_interface == 'pool2': output_pytorch = torch.from_numpy(np.rollaxis( F.avg_pool2d(torch.from_numpy(np.rollaxis(input_pytorch_value, 3, 1)).to("cuda"), kernel_size=(2, 2), stride=(2, 2)).to("cpu").numpy(), 1, 4)).to("cuda") elif target_interface == 'relu1': output_pytorch = F.relu(input_pytorch.to("cuda")) elif target_interface == 'dense1': output_pytorch = None elif target_interface == 'sigmoid1': output_pytorch = F.sigmoid(input_pytorch.to("cuda")) elif target_interface == 'tanh1': output_pytorch = F.tanh(input_pytorch.to("cuda")) else: output_pytorch = None output_pytorch_value = output_pytorch.to("cpu").numpy() return output_pytorch_value, input_pytorch_value, output_pytorch_cpu_value, input_pytorch_cpu_value
def batch_global_rigid_transformation(Rs, Js, parent, rotate_base=False, betas_logscale=None, opts=None): """ Computes absolute joint locations given pose. rotate_base: if True, rotates the global rotation by 90 deg in x axis. if False, this is the original SMPL coordinate. Args: Rs: N x 24 x 3 x 3 rotation vector of K joints Js: N x 24 x 3, joint locations before posing parent: 24 holding the parent id for each index Returns new_J : `Tensor`: N x 24 x 3 location of absolute joints A : `Tensor`: N x 24 4 x 4 relative joint transformations for LBS. """ if rotate_base: print('Flipping the SMPL coordinate frame!!!!') rot_x = torch.Tensor([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) rot_x = torch.reshape(torch.repeat(rot_x, [N, 1]), [N, 3, 3]) # In tf it was tile root_rotation = torch.matmul(Rs[:, 0, :, :], rot_x) else: root_rotation = Rs[:, 0, :, :] # Now Js is N x 24 x 3 x 1 Js = Js.unsqueeze(-1) N = Rs.shape[0] Js_orig = Js.clone() scaling_factors = torch.ones(N, parent.shape[0], 3).to(Rs.device) if betas_logscale is not None: leg_joints = list(range(7, 11)) + list(range(11, 15)) + list( range(17, 21)) + list(range(21, 25)) tail_joints = list(range(25, 32)) ear_joints = [33, 34] beta_scale_mask = torch.zeros(35, 3, 6).to(betas_logscale.device) beta_scale_mask[leg_joints, [2], [0]] = 1.0 # Leg lengthening beta_scale_mask[leg_joints, [0], [1]] = 1.0 # Leg fatness beta_scale_mask[leg_joints, [1], [1]] = 1.0 # Leg fatness beta_scale_mask[tail_joints, [0], [2]] = 1.0 # Tail lengthening beta_scale_mask[tail_joints, [1], [3]] = 1.0 # Tail fatness beta_scale_mask[tail_joints, [2], [3]] = 1.0 # Tail fatness beta_scale_mask[ear_joints, [1], [4]] = 1.0 # Ear y beta_scale_mask[ear_joints, [2], [5]] = 1.0 # Ear z beta_scale_mask = torch.transpose(beta_scale_mask.reshape(35 * 3, 6), 0, 1) betas_scale = torch.exp(betas_logscale @ beta_scale_mask) scaling_factors = betas_scale.reshape(-1, 35, 3) scale_factors_3x3 = torch.diag_embed(scaling_factors, dim1=-2, dim2=-1) def make_A(R, t): # Rs is N x 3 x 3, ts is N x 3 x 1 R_homo = torch.nn.functional.pad(R, (0, 0, 0, 1, 0, 0)) t_homo = torch.cat([t, torch.ones([N, 1, 1]).to(Rs.device)], 1) return torch.cat([R_homo, t_homo], 2) A0 = make_A(root_rotation, Js[:, 0]) results = [A0] for i in range(1, parent.shape[0]): j_here = Js[:, i] - Js[:, parent[i]] s_par_inv = torch.inverse(scale_factors_3x3[:, parent[i]]) rot = Rs[:, i] s = scale_factors_3x3[:, i] rot_new = s_par_inv @ rot @ s A_here = make_A(rot_new, j_here) res_here = torch.matmul(results[parent[i]], A_here) results.append(res_here) # 10 x 24 x 4 x 4 results = torch.stack(results, dim=1) # scale updates new_J = results[:, :, :3, 3] # --- Compute relative A: Skinning is based on # how much the bone moved (not the final location of the bone) # but (final_bone - init_bone) # --- Js_w0 = torch.cat([Js_orig, torch.zeros([N, 35, 1, 1]).to(Rs.device)], 2) init_bone = torch.matmul(results, Js_w0) # Append empty 4 x 3: init_bone = torch.nn.functional.pad(init_bone, (3, 0, 0, 0, 0, 0, 0, 0)) A = results - init_bone return new_J, A
def forward(self, graph, feat, weight=None): r"""Compute graph convolution. Notes ----- * Input shape: :math:`(N, *, \text{in_feats})` where * means any number of additional dimensions, :math:`N` is the number of nodes. * Output shape: :math:`(N, *, \text{out_feats})` where all but the last dimension are the same shape as the input. * Weight shape: :math:`(\text{in_feats}, \text{out_feats})`. Parameters ---------- graph : DGLGraph The graph. feat : torch.Tensor The input feature weight : torch.Tensor, optional Optional external weight tensor. Returns ------- torch.Tensor The output feature """ with graph.local_scope(): if self._norm == 'both': degs = graph.out_degrees().to(feat.device).float().clamp(min=1) norm = th.pow(degs, -0.5) shp = norm.shape + (1, ) * (feat.dim() - 1) norm = th.reshape(norm, shp) feat = feat * norm weight = self.weight if self._in_feats > self._out_feats and False: # mult W first to reduce the feature size for aggregation. if weight is not None: assert feat.shape[-1] == weight.shape[ -2], "feat shape is {}, weight shape is {}".format( feat.shape, weight.shape) feat = th.matmul(feat, weight) feat = self.apply_allgather(feat) graph.srcdata['h'] = feat graph.update_all(fn.copy_src(src='h', out='m'), fn.sum(msg='m', out='h')) rst = graph.dstdata['h'] rst = rst[:self._local_n_nodes, :] else: # aggregate first then mult W feat = self.apply_allgather(feat) graph.srcdata['h'] = feat graph.update_all(fn.copy_src(src='h', out='m'), fn.sum(msg='m', out='h')) rst = graph.dstdata['h'] rst = rst[:self._local_n_nodes, :] if weight is not None: rst = th.matmul(rst, weight) if self._norm != 'none': degs = graph.in_degrees().to(feat.device).float().clamp(min=1) if self._norm == 'both': norm = th.pow(degs, -0.5) else: norm = 1.0 / degs shp = norm.shape + (1, ) * (feat.dim() - 1) norm = th.reshape(norm, shp) rst = rst * norm if self.bias is not None: rst = rst + self.bias if self._comm_net: rst += th.matmul(feat[:self._local_n_nodes], self.comm_net_w) if self._activation is not None: rst = self._activation(rst) return rst
def forward(self, batch_data): images_in, boxes_in, bboxes_num_in = batch_data # read config parameters B = images_in.shape[0] T = images_in.shape[1] H, W = self.cfg.image_size OH, OW = self.cfg.out_size MAX_N = self.cfg.num_boxes NFB = self.cfg.num_features_boxes NFR, NFG = self.cfg.num_features_relation, self.cfg.num_features_gcn D = self.cfg.emb_features K = self.cfg.crop_size[0] if not self.training: B = B * 3 T = T // 3 images_in.reshape((B, T) + images_in.shape[2:]) boxes_in.reshape((B, T) + boxes_in.shape[2:]) bboxes_num_in.reshape((B, T)) # Reshape the input data images_in_flat = torch.reshape(images_in, (B * T, 3, H, W)) #B*T, 3, H, W boxes_in = boxes_in.reshape(B * T, MAX_N, 4) # Use backbone to extract features of images_in # Pre-precess first images_in_flat = prep_images(images_in_flat) outputs = self.backbone(images_in_flat) # Build multiscale features features_multiscale = [] for features in outputs: if features.shape[2:4] != torch.Size([OH, OW]): features = F.interpolate(features, size=(OH, OW), mode='bilinear', align_corners=True) features_multiscale.append(features) features_multiscale = torch.cat(features_multiscale, dim=1) #B*T, D, OH, OW boxes_in_flat = torch.reshape(boxes_in, (B * T * MAX_N, 4)) #B*T*MAX_N, 4 boxes_idx = [ i * torch.ones(MAX_N, dtype=torch.int) for i in range(B * T) ] boxes_idx = torch.stack(boxes_idx).to( device=boxes_in.device) # B*T, MAX_N boxes_idx_flat = torch.reshape(boxes_idx, (B * T * MAX_N, )) #B*T*MAX_N, # RoI Align boxes_in_flat.requires_grad = False boxes_idx_flat.requires_grad = False boxes_features_all = self.roi_align( features_multiscale, boxes_in_flat, boxes_idx_flat) #B*T*MAX_N, D, K, K, boxes_features_all = boxes_features_all.reshape(B * T, MAX_N, -1) #B*T,MAX_N, D*K*K # Embedding boxes_features_all = self.fc_emb_1( boxes_features_all) # B*T,MAX_N, NFB boxes_features_all = self.nl_emb_1(boxes_features_all) boxes_features_all = F.relu(boxes_features_all) boxes_features_all = boxes_features_all.reshape(B, T, MAX_N, NFB) boxes_in = boxes_in.reshape(B, T, MAX_N, 4) actions_scores = [] activities_scores = [] bboxes_num_in = bboxes_num_in.reshape(B, T) #B,T, for b in range(B): N = bboxes_num_in[b][0] boxes_features = boxes_features_all[b, :, :N, :].reshape( 1, T * N, NFB) #1,T,N,NFB boxes_positions = boxes_in[b, :, :N, :].reshape(T * N, 4) #T*N, 4 # GCN graph modeling for i in range(len(self.gcn_list)): graph_boxes_features, relation_graph = self.gcn_list[i]( boxes_features, boxes_positions) # cat graph_boxes_features with boxes_features boxes_features = boxes_features.reshape(1, T * N, NFB) boxes_states = graph_boxes_features + boxes_features #1, T*N, NFG boxes_states = self.dropout_global(boxes_states) NFS = NFG boxes_states = boxes_states.reshape(T, N, NFS) # Predict actions actn_score = self.fc_actions(boxes_states) #T,N, actn_num # Predict activities boxes_states_pooled, _ = torch.max(boxes_states, dim=1) #T, NFS acty_score = self.fc_activities(boxes_states_pooled) #T, acty_num # GSN fusion actn_score = torch.mean(actn_score, dim=0).reshape(N, -1) #N, actn_num acty_score = torch.mean(acty_score, dim=0).reshape(1, -1) #1, acty_num actions_scores.append(actn_score) activities_scores.append(acty_score) actions_scores = torch.cat(actions_scores, dim=0) #ALL_N,actn_num activities_scores = torch.cat(activities_scores, dim=0) #B,acty_num if not self.training: B = B // 3 actions_scores = torch.mean(actions_scores.reshape( -1, 3, actions_scores.shape[1]), dim=1) activities_scores = torch.mean(activities_scores.reshape(B, 3, -1), dim=1).reshape(B, -1) # print(actions_scores.shape) # print(activities_scores.shape) return actions_scores, activities_scores
def compute_loss(self, inferences, labels, regs): labels = torch.reshape(labels, [-1, 1]) loss = F.mse_loss(inferences, labels) return loss + regs
def attack_single_run_targeted(self, x, y=None, use_rand_start=False): """ :param x: clean images :param y: clean labels, if None we use the predicted labels """ if self.device is None: self.device = x.device self.orig_dim = list(x.shape[1:]) self.ndims = len(self.orig_dim) x = x.detach().clone().float().to(self.device) # assert next(self.predict.parameters()).device == x.device y_pred = self._get_predicted_label(x) if y is None: y = y_pred.detach().clone().long().to(self.device) else: y = y.detach().clone().long().to(self.device) pred = y_pred == y corr_classified = pred.float().sum() if self.verbose: print('Clean accuracy: {:.2%}'.format(pred.float().mean())) if pred.sum() == 0: return x pred = self.check_shape(pred.nonzero().squeeze()) output = self.predict(x) la_target = output.sort(dim=-1)[1][:, -self.target_class] startt = time.time() # runs the attack only on correctly classified points im2 = x[pred].detach().clone() la2 = y[pred].detach().clone() la_target2 = la_target[pred].detach().clone() if len(im2.shape) == self.ndims: im2 = im2.unsqueeze(0) bs = im2.shape[0] u1 = torch.arange(bs) adv = im2.clone() adv_c = x.clone() res2 = 1e10 * torch.ones([bs]).to(self.device) res_c = torch.zeros([x.shape[0]]).to(self.device) x1 = im2.clone() x0 = im2.clone().reshape([bs, -1]) counter_restarts = 0 while counter_restarts < 1: if use_rand_start: if self.norm == 'Linf': t = 2 * torch.rand(x1.shape).to(self.device) - 1 x1 = im2 + (torch.min( res2, self.eps * torch.ones(res2.shape).to(self.device) ).reshape([-1, *[1] * self.ndims])) * t / (t.reshape([ t.shape[0], -1 ]).abs().max(dim=1, keepdim=True)[0].reshape( [-1, *[1] * self.ndims])) * .5 elif self.norm == 'L2': t = torch.randn(x1.shape).to(self.device) x1 = im2 + (torch.min( res2, self.eps * torch.ones(res2.shape).to(self.device) ).reshape([-1, *[1] * self.ndims])) * t / ( (t**2).view(t.shape[0], -1).sum(dim=-1).sqrt().view( t.shape[0], *[1] * self.ndims)) * .5 elif self.norm == 'L1': t = torch.randn(x1.shape).to(self.device) x1 = im2 + (torch.min( res2, self.eps * torch.ones(res2.shape).to(self.device) ).reshape([-1, *[1] * self.ndims])) * t / (t.abs().view( t.shape[0], -1).sum(dim=-1).view( t.shape[0], *[1] * self.ndims)) / 2 x1 = x1.clamp(0.0, 1.0) counter_iter = 0 while counter_iter < self.n_iter: with torch.no_grad(): df, dg = self.get_diff_logits_grads_batch_targeted( x1, la2, la_target2) if self.norm == 'Linf': dist1 = df.abs() / (1e-12 + dg.abs().view( dg.shape[0], dg.shape[1], -1).sum(dim=-1)) elif self.norm == 'L2': dist1 = df.abs() / (1e-12 + (dg**2).view( dg.shape[0], dg.shape[1], -1).sum(dim=-1).sqrt()) elif self.norm == 'L1': dist1 = df.abs() / (1e-12 + dg.abs().reshape( [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) else: raise ValueError('norm not supported') ind = dist1.min(dim=1)[1] # print(ind) dg2 = dg[u1, ind] b = (-df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1).sum(dim=-1)) w = dg2.reshape([bs, -1]) if self.norm == 'Linf': d3 = self.projection_linf( torch.cat((x1.reshape([bs, -1]), x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) elif self.norm == 'L2': d3 = self.projection_l2( torch.cat((x1.reshape([bs, -1]), x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) elif self.norm == 'L1': d3 = self.projection_l1( torch.cat((x1.reshape([bs, -1]), x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) d1 = torch.reshape(d3[:bs], x1.shape) d2 = torch.reshape(d3[-bs:], x1.shape) if self.norm == 'Linf': a0 = d3.abs().max(dim=1, keepdim=True)[0] \ .view(-1, *[1] * self.ndims) elif self.norm == 'L2': a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt() \ .view(-1, *[1] * self.ndims) elif self.norm == 'L1': a0 = d3.abs().sum(dim=1, keepdim=True) \ .view(-1, *[1] * self.ndims) a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape).to(self.device)) a1 = a0[:bs] a2 = a0[-bs:] alpha = torch.min( torch.max(a1 / (a1 + a2), torch.zeros(a1.shape).to(self.device)), self.alpha_max * torch.ones(a1.shape).to(self.device)) x1 = ((x1 + self.eta * d1) * (1 - alpha) + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) is_adv = self._get_predicted_label(x1) != la2 if is_adv.sum() > 0: ind_adv = is_adv.nonzero().squeeze() ind_adv = self.check_shape(ind_adv) if self.norm == 'Linf': t = (x1[ind_adv] - im2[ind_adv]).reshape( [ind_adv.shape[0], -1]).abs().max(dim=1)[0] elif self.norm == 'L2': t = ((x1[ind_adv] - im2[ind_adv]) ** 2) \ .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() elif self.norm == 'L1': t = (x1[ind_adv] - im2[ind_adv]) \ .abs().view(ind_adv.shape[0], -1).sum(dim=-1) adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]). \ float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] \ * (t >= res2[ind_adv]).float().reshape( [-1, *[1] * self.ndims]) res2[ind_adv] = t * (t < res2[ind_adv]).float() \ + res2[ind_adv] * (t >= res2[ind_adv]).float() x1[ind_adv] = im2[ind_adv] + (x1[ind_adv] - im2[ind_adv]) * self.beta counter_iter += 1 counter_restarts += 1 ind_succ = res2 < 1e10 if self.verbose: print('success rate: {:.0f}/{:.0f}'.format(ind_succ.float().sum(), corr_classified) + ' (on correctly classified points) in {:.1f} s'.format( time.time() - startt)) res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) adv_c[pred[ind_succ]] = adv[ind_succ].clone() return adv_c
optimizer = torch.optim.SGD([model.W, model.b], learning_rate) for epoch in range(epochs): model.loss(x_train, y_train).backward() # Compute loss gradients optimizer.step() # Perform optimization by adjusting W and b optimizer.zero_grad() # Clear gradients for next step # Print model variables and loss print("W = %s, b = %s, loss = %s" % (model.W, model.b, model.loss(x_train, y_train))) # Visualize result fig = plt.figure("NOT operator") plot1 = fig.add_subplot() plt.plot(x_train.detach(), y_train.detach(), 'o', label='$(\\hat x^{(i)},\\hat y^{(i)})$') plt.xlabel('x') plt.ylabel('y') out = torch.reshape(torch.tensor(np.linspace(0, 1, 100).tolist()), (-1, 1)) plot1.set_xticks([0, 1]) # x range from 0 to 1 plot1.set_yticks([0, 1]) # y range from 0 to 1 x, indices = torch.sort(out, 0) # Plot sigmoid regression curve. plt.plot(x, model.f(x).detach()) plt.show()
optimizer = torch.optim.SGD(model.parameters(), lr=lr) #print('Number of model parameters: {}'.format( # len([p.data.nelement() for p in model.parameters()]))) #print(model) ###################################################Training the data######################################## for ep in range(num_epochs): epoch_file = open(filename, 'a') epoch_file.write('Epoch {}/{} --- \n'.format(ep + 1, num_epochs)) epoch_file.close() for i, (text, label) in enumerate(train_loader): #print(text[0,:10],label[0]) if torch.cuda.is_available(): text = torch.reshape(text, (batch_size, 1, 1024)) text = text.long() text = Variable(text.cuda()) label = Variable(label.cuda()) optimizer.zero_grad() predict = model.forward(text) temp = 0 loss = loss_function(predict, label) loss.backward() optimizer.step() k = 0.0 for i, (text, label) in enumerate(test_loader): if torch.cuda.is_available(): text = torch.reshape(text, (batch_size, 1, 1024)) text = text.long()
def _train(self, BATCH): obs = get_first_vector(BATCH.obs) # [T, B, S] obs_ = get_first_vector(BATCH.obs_) # [T, B, S] _timestep = obs.shape[0] _batchsize = obs.shape[1] predicted_obs_ = self._forward_dynamic_model(obs, BATCH.action) # [T, B, S] predicted_reward = self._reward_model(obs, BATCH.action) # [T, B, 1] predicted_done_dist = self._done_model(obs, BATCH.action) # [T, B, 1] _obs_loss = F.mse_loss(obs_, predicted_obs_) # todo _reward_loss = F.mse_loss(BATCH.reward, predicted_reward) _done_loss = -predicted_done_dist.log_prob(BATCH.done).mean() wm_loss = _obs_loss + _reward_loss + _done_loss self._wm_oplr.optimize(wm_loss) obs = th.reshape(obs, (_timestep * _batchsize, -1)) # [T*B, S] obs_ = th.reshape(obs_, (_timestep * _batchsize, -1)) # [T*B, S] actions = th.reshape(BATCH.action, (_timestep * _batchsize, -1)) # [T*B, A] rewards = th.reshape(BATCH.reward, (_timestep * _batchsize, -1)) # [T*B, 1] dones = th.reshape(BATCH.done, (_timestep * _batchsize, -1)) # [T*B, 1] rollout_rewards = [rewards] rollout_dones = [dones] r_obs_ = obs_ _r_obs = deepcopy(BATCH.obs_) r_done = (1. - dones) for _ in range(self._roll_out_horizon): r_obs = r_obs_ _r_obs.vector.vector_0 = r_obs if self.is_continuous: action_target = self.actor.t(_r_obs) # [T*B, A] if self.use_target_action_noise: r_action = self.target_noised_action( action_target) # [T*B, A] else: target_logits = self.actor.t(_r_obs) # [T*B, A] target_cate_dist = td.Categorical(logits=target_logits) target_pi = target_cate_dist.sample() # [T*B,] r_action = F.one_hot(target_pi, self.a_dim).float() # [T*B, A] r_obs_ = self._forward_dynamic_model(r_obs, r_action) # [T*B, S] r_reward = self._reward_model(r_obs, r_action) # [T*B, 1] r_done = r_done * (1. - self._done_model(r_obs, r_action).sample() ) # [T*B, 1] rollout_rewards.append(r_reward) # [H+1, T*B, 1] rollout_dones.append(r_done) # [H+1, T*B, 1] _r_obs.vector.vector_0 = obs q = self.critic(_r_obs, actions) # [T*B, 1] _r_obs.vector.vector_0 = r_obs_ q_target = self.critic.t(_r_obs, r_action) # [T*B, 1] dc_r = rewards for t in range(1, self._roll_out_horizon): dc_r += (self.gamma**t) * (rollout_rewards[t] * rollout_dones[t]) dc_r += (self.gamma**self._roll_out_horizon) * rollout_dones[ self._roll_out_horizon] * q_target # [T*B, 1] td_error = dc_r - q # [T*B, 1] q_loss = td_error.square().mean() # 1 self.critic_oplr.optimize(q_loss) # train actor if self.is_continuous: mu = self.actor(BATCH.obs, begin_mask=BATCH.begin_mask) # [T, B, A] else: logits = self.actor(BATCH.obs, begin_mask=BATCH.begin_mask) # [T, B, A] logp_all = logits.log_softmax(-1) # [T, B, A] gumbel_noise = td.Gumbel(0, 1).sample(logp_all.shape) # [T, B, A] _pi = ((logp_all + gumbel_noise) / self.discrete_tau).softmax( -1) # [T, B, A] _pi_true_one_hot = F.one_hot(_pi.argmax(-1), self.a_dim).float() # [T, B, A] _pi_diff = (_pi_true_one_hot - _pi).detach() # [T, B, A] mu = _pi_diff + _pi # [T, B, A] q_actor = self.critic(BATCH.obs, mu, begin_mask=BATCH.begin_mask) # [T, B, 1] actor_loss = -q_actor.mean() # 1 self.actor_oplr.optimize(actor_loss) return th.ones_like(BATCH.reward), { 'LEARNING_RATE/wm_lr': self._wm_oplr.lr, 'LEARNING_RATE/actor_lr': self.actor_oplr.lr, 'LEARNING_RATE/critic_lr': self.critic_oplr.lr, 'LOSS/wm_loss': wm_loss, 'LOSS/actor_loss': actor_loss, 'LOSS/critic_loss': q_loss, 'Statistics/q_min': q.min(), 'Statistics/q_mean': q.mean(), 'Statistics/q_max': q.max() }
def forward(self, x): """Applies network layers and ops on input image(s) x. Args: x: input image or batch of images. Shape: [batch,3,300,300]. Return: Depending on phase: test: Variable(tensor) of output class label predictions, confidence score, and corresponding location predictions for each object detected. Shape: [batch,topk,7] train: list of concat outputs from: 1: confidence layers, Shape: [batch*num_priors,num_classes] 2: localization layers, Shape: [batch,num_priors*4] 3: priorbox layers, Shape: [2,num_priors*4] """ size = x.size()[2:] batch_size = x.shape[0] sources = list() loc = list() conf = list() x = self.conv_top(x) s = self.L2Norm3_3(x) sources.append(s) patches = self.unfold(x) patches = torch.cat(torch.unbind(patches, dim=2), dim=0) patches = torch.reshape(patches, (-1, 4, 8, 8)) output_x = int((x.shape[2] - 8) / 4 + 1) output_y = int((x.shape[3] - 8) / 4 + 1) rnnX = self.rnn_model(patches, int(batch_size) * output_x * output_y) x = torch.stack(torch.split(rnnX, split_size_or_sections=int(batch_size), dim=0), dim=2) x = F.fold(x, kernel_size=(1, 1), output_size=(output_x, output_y)) x = F.pad(x, (0, 1, 0, 1), mode='replicate') for k in range(4): x = self.mob[k](x) s = self.L2Norm4_3(x) sources.append(s) for k in range(4, 8): x = self.mob[k](x) s = self.L2Norm5_3(x) sources.append(s) for k in range(8, 10): x = self.mob[k](x) sources.append(x) for k in range(10, 11): x = self.mob[k](x) sources.append(x) for k in range(11, 12): x = self.mob[k](x) sources.append(x) # apply multibox head to source layers loc_x = self.loc[0](sources[0]) conf_x = self.conf[0](sources[0]) loc_x = self.loc[1](loc_x) conf_x = self.conf[1](conf_x) max_conf, _ = torch.max(conf_x[:, 0:3, :, :], dim=1, keepdim=True) conf_x = torch.cat((max_conf, conf_x[:, 3:, :, :]), dim=1) loc.append(loc_x.permute(0, 2, 3, 1).contiguous()) conf.append(conf_x.permute(0, 2, 3, 1).contiguous()) for i in range(1, len(sources)): x = sources[i] conf.append(self.conf[i + 1](x).permute(0, 2, 3, 1).contiguous()) loc.append(self.loc[i + 1](x).permute(0, 2, 3, 1).contiguous()) features_maps = [] for i in range(len(loc)): feat = [] feat += [loc[i].size(1), loc[i].size(2)] features_maps += [feat] self.priorbox = PriorBox(size, features_maps, cfg) self.priors = self.priorbox.forward() loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1) conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1) if self.phase == 'test': output = detect_function( cfg, loc.view(loc.size(0), -1, 4), # loc preds self.softmax(conf.view(conf.size(0), -1, self.num_classes)), # conf preds self.priors.type(type(x.data)) # default boxes ) else: output = (loc.view(loc.size(0), -1, 4), conf.view(conf.size(0), -1, self.num_classes), self.priors) return output, loc, conf
def forward(self, input): input = torch.reshape( input, (input.size(0), 1, input.size(2) * input.size(3))) self.X = input return super().forward(input)
# Get the most probable last output index next_word_encoded = net_out[:, -1, :] closest_value = np.linalg.norm( (X - next_word_encoded.to('cpu').numpy()[0]), axis=1) closest_word = emb.index[np.argmin(closest_value)] state += ' ' + closest_word # Print the seed letters #print(state, end=' ', flush=True) #%% Generate sonnet new_line_count = 0 tot_char_count = 0 while True: with torch.no_grad(): # No need to track the gradients # The new network input is the one hot encoding of the last chosen letter net_input = ds.encode_text(encoder, state.lower(), emb) net_input = torch.reshape( torch.tensor(net_input).float(), (1, -1, 50)) #print(net_input.shape) #net_input = net_input.unsqueeze(0) # Forward pass #net_input = net_input.to(device) net_out, net_state = net(net_input, net_state) # Get the most probable letter index closest_value = np.linalg.norm( (X - net_out[:, -1, :].to('cpu').numpy()[0]), axis=1) closest_word = emb.index[np.argmin(closest_value)] state += ' ' + closest_word #print(closest_word, end=' ', flush=True) # Count total letters tot_char_count += 1 # Count new lines
def drop_block_2d( x, drop_prob: float = 0.1, block_size: int = 7, gamma_scale: float = 1.0, with_noise: bool = False, inplace: bool = False, batchwise: bool = False, ): """DropBlock. See https://arxiv.org/pdf/1810.12890.pdf DropBlock with an experimental gaussian noise option. This layer has been tested on a few training runs with success, but needs further validation and possibly optimization for lower runtime impact. """ B, C, H, W = x.shape total_size = W * H clipped_block_size = min(block_size, min(W, H)) # seed_drop_rate, the gamma parameter gamma = (gamma_scale * drop_prob * total_size / clipped_block_size**2 / ((W - block_size + 1) * (H - block_size + 1))) # Forces the block to be inside the feature map. w_i, h_i = torch.meshgrid( torch.arange(W).to(x.device), torch.arange(H).to(x.device)) valid_block = ((w_i >= clipped_block_size // 2) & (w_i < W - (clipped_block_size - 1) // 2)) & ( (h_i >= clipped_block_size // 2) & (h_i < H - (clipped_block_size - 1) // 2)) valid_block = torch.reshape(valid_block, (1, 1, H, W)).to(dtype=x.dtype) if batchwise: # one mask for whole batch, quite a bit faster uniform_noise = torch.rand((1, C, H, W), dtype=x.dtype, device=x.device) else: uniform_noise = torch.rand_like(x) block_mask = ((2 - gamma - valid_block + uniform_noise) >= 1).to( dtype=x.dtype) block_mask = -F.max_pool2d( -block_mask, kernel_size=clipped_block_size, # block_size, stride=1, padding=clipped_block_size // 2, ) if with_noise: normal_noise = (torch.randn( (1, C, H, W), dtype=x.dtype, device=x.device) if batchwise else torch.randn_like(x)) if inplace: x.mul_(block_mask).add_(normal_noise * (1 - block_mask)) else: x = x * block_mask + normal_noise * (1 - block_mask) else: normalize_scale = ( block_mask.numel() / block_mask.to(dtype=torch.float32).sum().add(1e-7)).to(x.dtype) if inplace: x.mul_(block_mask * normalize_scale) else: x = x * block_mask * normalize_scale return x
def train(self): """ Implement your training algorithm here """ for episode in range(self.episodes): state = self.env.reset() state = torch.reshape(tensor(state, dtype=torch.float32), [1, 84, 84, 4]).permute(0, 3, 1, 2).to( self.device) done = False episode_reward = [] episode_loss = [] # save network if episode % self.model_save_interval == 0: save_path = self.model_save_path + '/' + self.run_name + '_' + str(episode) + '.pt' torch.save(self.q_network.state_dict(), save_path) print('Successfully saved: ' + save_path) while not done: # update target network if self.step % self.network_update_interval == 0: print('Updating target network') self.target_network.load_state_dict(self.q_network.state_dict()) if self.step > len(self.replay_memory): self.epsilon = max(self.final_epsilon, self.initial_epsilon - self.epsilon_step * self.step) if self.epsilon > self.final_epsilon: self.mode = 'Explore' else: self.mode = 'Exploit' action, q = self.make_action(state, 0, test=False) next_state, reward, done, _ = self.env.step(action) next_state = torch.reshape(tensor(next_state, dtype=torch.float32), [1, 84, 84, 4]).permute(0, 3, 1, 2).to( self.device) self.push((state, torch.tensor([int(action)]), torch.tensor([reward], device=self.device), next_state, torch.tensor([done], dtype=torch.float32))) episode_reward.append(reward) self.step += 1 state = next_state # train network if self.step >= self.start_to_learn and self.step % self.network_train_interval == 0: loss = self.optimize_network() episode_loss.append(loss) if done: print('Episode:', episode, ' | Steps:', self.step, ' | Eps: ', self.epsilon, ' | Reward: ', sum(episode_reward), ' | Avg Reward: ', np.mean(self.last_n_rewards), ' | Loss: ', np.mean(episode_loss), ' | Intrinsic Reward: ', sum(self.intrinsic_episode_reward), ' | Avg Intrinsic Reward: ', np.mean(self.last_n_intrinsic_rewards), ' | Mode: ', self.mode) print('Episode:', episode, ' | Steps:', self.step, ' | Eps: ', self.epsilon, ' | Reward: ', sum(episode_reward), ' | Avg Reward: ', np.mean(self.last_n_rewards), ' | Loss: ', np.mean(episode_loss), ' | Intrinsic Reward: ', sum(self.intrinsic_episode_reward), ' | Avg Intrinsic Reward: ', np.mean(self.last_n_intrinsic_rewards), ' | Mode: ', self.mode, file=self.log_file) self.log_summary(episode, episode_loss, episode_reward) self.last_n_rewards.append(sum(episode_reward)) self.last_n_intrinsic_rewards.append(sum(self.intrinsic_episode_reward)) episode_reward.clear() episode_loss.clear() self.intrinsic_episode_reward.clear()
def forward(self, batch_data): images_in, boxes_in = batch_data # read config parameters B = images_in.shape[0] T = images_in.shape[1] H, W = self.cfg.image_size OH, OW = self.cfg.out_size N = self.cfg.num_boxes NFB = self.cfg.num_features_boxes NFR, NFG = self.cfg.num_features_relation, self.cfg.num_features_gcn NG = self.cfg.num_graph D = self.cfg.emb_features K = self.cfg.crop_size[0] if not self.training: B = B * 3 T = T // 3 images_in.reshape((B, T) + images_in.shape[2:]) boxes_in.reshape((B, T) + boxes_in.shape[2:]) # Reshape the input data images_in_flat = torch.reshape(images_in, (B * T, 3, H, W)) # B*T, 3, H, W boxes_in_flat = torch.reshape(boxes_in, (B * T * N, 4)) # B*T*N, 4 boxes_idx = [i * torch.ones(N, dtype=torch.int) for i in range(B * T)] boxes_idx = torch.stack(boxes_idx).to(device=boxes_in.device) # B*T, N boxes_idx_flat = torch.reshape(boxes_idx, (B * T * N, )) # B*T*N, # Use backbone to extract features of images_in # Pre-precess first images_in_flat = prep_images(images_in_flat) outputs = self.backbone(images_in_flat) # Build features assert outputs[0].shape[2:4] == torch.Size([OH, OW]) features_multiscale = [] for features in outputs: if features.shape[2:4] != torch.Size([OH, OW]): features = F.interpolate(features, size=(OH, OW), mode='bilinear', align_corners=True) features_multiscale.append(features) features_multiscale = torch.cat(features_multiscale, dim=1) # B*T, D, OH, OW # RoI Align boxes_in_flat.requires_grad = False boxes_idx_flat.requires_grad = False boxes_features = self.roi_align(features_multiscale, boxes_in_flat, boxes_idx_flat) # B*T*N, D, K, K, boxes_features = boxes_features.reshape(B, T, N, -1) # B,T,N, D*K*K # Embedding boxes_features = self.fc_emb_1(boxes_features) # B,T,N, NFB boxes_features = self.nl_emb_1(boxes_features) boxes_features = F.relu(boxes_features) # GCN graph_boxes_features = boxes_features.reshape(B, T * N, NFG) # visual_info=[] for i in range(len(self.gcn_list)): graph_boxes_features, relation_graph = self.gcn_list[i]( graph_boxes_features, boxes_in_flat) # visual_info.append(relation_graph.reshape(B,T,N,N)) # fuse graph_boxes_features with boxes_features graph_boxes_features = graph_boxes_features.reshape(B, T, N, NFG) boxes_features = boxes_features.reshape(B, T, N, NFB) # boxes_states= torch.cat( [graph_boxes_features,boxes_features],dim=3) #B, T, N, NFG+NFB boxes_states = graph_boxes_features + boxes_features boxes_states = self.dropout_global(boxes_states) NFS = NFG # Predict actions boxes_states_flat = boxes_states.reshape(-1, NFS) # B*T*N, NFS actions_scores = self.fc_actions(boxes_states_flat) # B*T*N, actn_num # Predict activities boxes_states_pooled, _ = torch.max(boxes_states, dim=2) boxes_states_pooled_flat = boxes_states_pooled.reshape(-1, NFS) activities_scores = self.fc_activities( boxes_states_pooled_flat) # B*T, acty_num # Temporal fusion actions_scores = actions_scores.reshape(B, T, N, -1) actions_scores = torch.mean(actions_scores, dim=1).reshape(B * N, -1) activities_scores = activities_scores.reshape(B, T, -1) activities_scores = torch.mean(activities_scores, dim=1).reshape(B, -1) if not self.training: B = B // 3 actions_scores = torch.mean(actions_scores.reshape(B, 3, N, -1), dim=1).reshape(B * N, -1) activities_scores = torch.mean(activities_scores.reshape(B, 3, -1), dim=1).reshape(B, -1) return actions_scores, activities_scores
seed = 1234 ##### ### Shuffling the list to randomize the data random.shuffle(training_files) ###### N_batches = 10 for epoch in range(N_epochs): loss_sum = 0 err_sum = 0 acc_sum = 0 for i in range(N_batches): input_data, labels = create_batches_rnd(batch_size, training_files, N_train_files, window_len, 0.2) inputs = torch.reshape(input_data, (batch_size, 1, input_shape[0], input_shape[1])) inputs = inputs.to(device) labels = labels.to(device) output = model(inputs) loss = criterion(output, labels.long()) prediction = torch.max(output, dim=1)[1] err = torch.mean((prediction != labels.long()).float()) # Backward and optimize optimizer.zero_grad() loss.backward() optimizer.step() #print(loss.item()) loss_sum = loss_sum + loss.detach() err_sum = err_sum + err.detach() acc_sum = acc_sum + torch.mean((prediction == labels.long()).float()) loss_tot = loss_sum / N_batches
def unit_test_regressR(self): print('Unit test regressR') # Only regress G0, the rest can be solved numerically. real_thetas = torch.from_numpy((np.random.rand(32, pose_size) - 0.5) * 1.5)\ .type(torch.float64).to(self.device) ''' Fix global rotation, change local rotation ''' # Test if change R[23] affects joint 23, and the shape of finger tips? # Check passed, change leaf rotation does not affect leaf joints, but do affect the body shapes # Consider adding another control point at head to determine head rotation. undefined = [ 30, 31, 32, 33, 34, 35, 45, 46, 47, 66, 67, 68, 69, 70, 71 ] index = torch.zeros(72, dtype=torch.int, device=self.device) index[undefined] = 1 for j in range(1, 32): real_thetas[j] = torch.where(index == 0, real_thetas[0], real_thetas[j]) #print('thetas:', real_thetas) betas = torch.from_numpy(np.zeros((32, beta_size))) \ .type(torch.float64).to(self.device) trans = torch.from_numpy(np.zeros( (32, 3))).type(torch.float64).to(self.device) meshes, joints = self.forward(betas, real_thetas, trans) #reg_G = self.regressG(joints) v_shaped = torch.tensordot(betas, self.shapedirs, dims=([1], [2])) + self.v_template J = torch.matmul(self.J_regressor, v_shaped) G, R_cube_big = self.theta2G(real_thetas, J) # pre-calculate G terms for skinning # print(G[0,0]) # R00 = G[0,0,:3,:3] # print('j0:',self.J0[0]) # j0_ = torch.matmul( # torch.eye(3, dtype=torch.float64, device =self.device) - R00, # self.J0[0] # ) # print('(I-R_0)J0:',j0_) # What if we directly apply G on J? J_1 = torch.cat((J, torch.ones( (32, J.shape[1], 1), dtype=torch.float64).to(self.device)), dim=2).reshape(32, -1, 4, 1) fake_joints = torch.matmul(G, J_1) fake_joints = torch.reshape(fake_joints, (32, -1, 4))[:, :, :3] #print('G_0J0:',fake_joints[0,0]) #print('real joint_0:',joints[0,0]) # Test if directly regress joints from G works... # 20190308: Good approximation, visually undiscernable. for i in range(32): model.write_obj(meshes[i].detach().cpu().numpy(), './joint_test_0308/real_{}.obj'.format(i)) np.savetxt('./joint_test_0308/real_{}.xyz'.format(i), joints[i].detach().cpu().numpy(), delimiter=' ') np.savetxt('./joint_test_0308/fake_{}.xyz'.format(i), fake_joints[i].detach().cpu().numpy(), delimiter=' ')
def main(): TRAIN_IMG_DIR = "data/images" VAL_IMG_DIR = "data/images" LABEL_DIR = "data/labels" BATCH_SIZE = 16 NUM_WORKERS = 2 PIN_MEMORY = True DEVICE = "cuda" if torch.cuda.is_available() else "cpu" LEARNING_RATE = 2e-4 WEIGHT_DECAY = 0.0 EPOCHS = 100 TRAIN = False LOAD_MODEL_FILE = "epoch_trained.pth.tar" S = 7 C = 20 B = 2 print(DEVICE) class Compose(object): def __init__(self, transforms): self.transforms = transforms def __call__(self, image, bboxes): for t in self.transforms: image = t(image) return image, bboxes # Types of preprocessing transforms we want to apply convert_transform = transforms.ToTensor() resize_transform = transforms.Resize((448, 448)) # normalize_transform = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform = Compose([convert_transform, resize_transform]) train_dataset = VOCDataset("data/train.csv", S=S, C=C, B=B, transform=transform, img_dir=VAL_IMG_DIR, label_dir=LABEL_DIR) train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True) val_dataset = VOCDataset("data/train.csv", S=S, C=C, B=B, transform=transform, img_dir=VAL_IMG_DIR, label_dir=LABEL_DIR) val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True) model = YoloV1(S=S, B=B, C=C).to(DEVICE) for child in model.resnet18.parameters(): child.requires_grad = True model.resnet18.fc.requires_grad = True optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) yolo_loss = YoloLoss(S=S, C=C, B=B) writer = SummaryWriter() # TRAINING if TRAIN: mean_loss_list = [] for epoch in tqdm(range(EPOCHS)): mean_loss = [] for batch_idx, (x, y) in enumerate(train_loader): x, y = x.to(DEVICE), y.to(DEVICE) y_p = model(x) # y = torch.flatten(y, start_dim=1, end_dim=3) y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B)) loss = yolo_loss(y_p, y) mean_loss.append(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() # Calculate average loss mean_loss = sum(mean_loss) / len(mean_loss) writer.add_scalar("Average Loss: ", mean_loss, epoch) print("Epoch:", epoch) print(f"Mean loss was {mean_loss}") mean_loss_list.append(mean_loss) # Save the model if True: checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } print("=> Saving checkpoint") torch.save(checkpoint, LOAD_MODEL_FILE) time.sleep(10) writer.close() plt.plot(mean_loss_list) plt.show() # VALIDATION else: print("=> Loading checkpoint") model.load_state_dict(torch.load(LOAD_MODEL_FILE)["state_dict"]) optimizer.load_state_dict(torch.load(LOAD_MODEL_FILE)["optimizer"]) model.train() # Sets model to training mode model.eval() # Sets model to evaluation mode with torch.no_grad(): # Use with evaluation mode for epoch in tqdm(range(1)): for i, (x, y) in enumerate(val_loader): x, y = x.to(DEVICE), y.to(DEVICE) y_p = model(x) x = x.to('cpu') y_p = y_p.to('cpu') y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B)) print("Rendering original labels (y to x)") render_batch(x, y) print("Rendering predicted labels (y_p to x)") render_batch(x, y_p)
def __class_predictor(self, feature_maps): convoutput = F.conv2d(feature_maps, self.class_predictor_weights) new_shape = convoutput.shape[0:1] + torch.Size( [self.num_classes, self.nboxes_per_pixel]) + convoutput.shape[2:4] # dont take softmax here, take logsoftmax in the loss return torch.reshape(convoutput, new_shape)
def forward(self, betas, thetas, trans, gR=None): """ Construct a compute graph that takes in parameters and outputs a tensor as model vertices. Face indices are also returned as a numpy ndarray. 20190128: Add batch support. 20190322: Extending forward compatiability with SMPLModelv3 Usage: --------- meshes, joints = forward(betas, thetas, trans): normal SMPL meshes, joints = forward(betas, thetas, trans, gR=gR): calling from SMPLModelv3, using gR to cache G terms, ignoring thetas Parameters: --------- thetas: an [N, 24 * 3] tensor indicating child joint rotation relative to parent joint. For root joint it's global orientation. Represented in a axis-angle format. betas: Parameter for model shape. A tensor of shape [N, 10] as coefficients of PCA components. Only 10 components were released by SMPL author. trans: Global translation tensor of shape [N, 3]. G, R_cube_big: (Added on 0322) Fix compatible issue when calling from v3 objects when calling this mode, theta must be set as None Return: ------ A 3-D tensor of [N * 6890 * 3] for vertices, and the corresponding [N * 24 * 3] joint positions. """ batch_num = betas.shape[0] v_shaped = torch.tensordot(betas, self.shapedirs, dims=([1], [2])) + self.v_template J = torch.matmul(self.J_regressor, v_shaped) if gR is not None: G, R_cube_big = self.gR2G(gR, J) elif thetas is not None: G, R_cube_big = self.theta2G( thetas, J) # pre-calculate G terms for skinning else: raise (RuntimeError( 'Either thetas or gR should be specified, but detected two Nonetypes' )) # (1) Pose shape blending (SMPL formula(9)) if self.simplify: v_posed = v_shaped else: R_cube = R_cube_big[:, 1:, :, :] I_cube = (torch.eye(3, dtype=self.data_type).unsqueeze(dim=0) + \ torch.zeros((batch_num, R_cube.shape[1], 3, 3), dtype=self.data_type)).to(self.device) lrotmin = (R_cube - I_cube).reshape(batch_num, -1) v_posed = v_shaped + torch.tensordot( lrotmin, self.posedirs, dims=([1], [2])) # (2) Skinning (W) T = torch.tensordot(G, self.weights, dims=([1], [1])).permute(0, 3, 1, 2) rest_shape_h = torch.cat((v_posed, torch.ones( (batch_num, v_posed.shape[1], 1), dtype=self.data_type).to(self.device)), dim=2) v = torch.matmul(T, torch.reshape(rest_shape_h, (batch_num, -1, 4, 1))) v = torch.reshape(v, (batch_num, -1, 4))[:, :, :3] result = v + torch.reshape(trans, (batch_num, 1, 3)) # estimate 3D joint locations #joints = torch.tensordot(result, self.joint_regressor, dims=([1], [0])).transpose(1, 2) joints = torch.tensordot(result, self.J_regressor.transpose(0, 1), dims=([1], [0])).transpose(1, 2) return result, joints
def prepare(feat): B, n_ch, n_voxels = feat.size() n_ch_1 = n_ch + 1 conv_tensor = np.tril(np.ones((n_ch + 1, n_ch + 1)), -1).T conv_tensor += np.diag([-i for i in range(n_ch + 1)]) conv_tensor = conv_tensor[:, 1:] conv_tensor = np.matmul( conv_tensor, np.sqrt(np.diag([1 / (d * (d + 1)) for d in range(1, n_ch + 1)]))) inv_std_dev = np.sqrt(2 / 3.) * (n_ch + 1) conv_tensor *= inv_std_dev conv_tensor = conv_tensor[:, :, np.newaxis] feat = F.conv1d(feat, torch.FloatTensor(conv_tensor).cuda()) feat_v = torch.round(feat / (n_ch + 1)) rem0 = feat_v * (n_ch + 1) index = torch.argsort(feat - rem0, dim=1, descending=True) rank = torch.argsort(index, dim=1, descending=False) rank = rank + torch.sum(feat_v, 1).unsqueeze(1).type( torch.cuda.LongTensor) add_minus = (rank < 0).type( torch.cuda.LongTensor) - (rank > n_ch).type(torch.cuda.LongTensor) add_minus *= (n_ch + 1) rank = rank + add_minus rem0 = rem0 + add_minus.type(torch.cuda.FloatTensor) y = (feat - rem0) / (n_ch + 1) v_sorted = torch.sort(y, dim=1, descending=False)[0] barycentric = v_sorted - torch.cat( [v_sorted[:, -1:] - 1., v_sorted[:, :-1]], 1) canonical = torch.cuda.FloatTensor([[i] * ((n_ch + 1) - i) + [i - (n_ch + 1)] * i for i in range(n_ch + 1)]) def _simple_hash(key): key = key.type(torch.cuda.DoubleTensor) hash_vector = np.floor( np.power(np.iinfo(np.int64).max, 1. / (n_ch + 2))) hash_vector = torch.pow(hash_vector, torch.arange(1, n_ch + 1)) hash_vector = hash_vector.type(torch.DoubleTensor).unsqueeze(0) hash_vector = hash_vector.cuda() if len(key.size()) == 3: hash_vector = hash_vector.unsqueeze(2) return torch.sum( key * hash_vector.repeat(key.size(0), 1, key.size(-1)), 1) if len(key.size()) == 2: return torch.sum(key * hash_vector.repeat(key.size(0), 1), 1) dic_hash_lattice = HashTable(n_ch, torch.cuda.DoubleTensor, 2**30) loc = [None] * (n_ch + 1) loc_hash = [None] * (n_ch + 1) for scit in range(n_ch + 1): loc[scit] = torch.gather( canonical[scit:scit + 1].unsqueeze(2).repeat( rank.size(0), 1, rank.size(2)), 1, rank[:, :-1]) loc[scit] += rem0[:, :-1] loc[scit] = loc[scit] loc_hash[scit] = _simple_hash(loc[scit]) loc[scit] = torch.reshape(loc[scit].permute(0, 2, 1), (-1, n_ch)) dic_hash_lattice.add_values(loc_hash[scit].view(-1), loc[scit]) dic_hash_lattice.filter_values() fused_loc = dic_hash_lattice.export_values() dic_hash_lattice.update_rank() indices = [None] * n_ch_1 blur_neighbours1 = [None] * n_ch_1 blur_neighbours2 = [None] * n_ch_1 default = torch.tensor(0).type(torch.LongTensor).cuda() for dit in range(n_ch_1): offset = [n_ch if i == dit else -1 for i in range(n_ch)] offset = torch.cuda.FloatTensor(offset) blur_neighbours1[dit] = dic_hash_lattice.get_rank( _simple_hash(fused_loc + offset).view(-1))[:, 0] blur_neighbours2[dit] = dic_hash_lattice.get_rank( _simple_hash(fused_loc - offset).view(-1))[:, 0] indices[dit] = dic_hash_lattice.get_rank( loc_hash[dit].view(-1)).view(B, n_voxels) return rank, barycentric, blur_neighbours1, blur_neighbours2, indices
def forward(self, inputs, input_frames=20, future_frames=1, output_frames=1, channels=1, teacher_forcing=False, scheduled_sampling_ratio=0): batch_size, input_frames, height = inputs.size() output_frames = input_frames inputs = torch.reshape(inputs, (batch_size, input_frames, channels, height)) """ Computation of Convolutional LSTM network. Arguments: ---------- inputs: a 5-th order tensor of size [batch_size, input_frames, input_channels, height, width] Input tensor (video) to the deep Conv-LSTM network. input_frames: int The number of input frames to the model. future_frames: int The number of future frames predicted by the model. output_frames: int The number of output frames returned by the model. teacher_forcing: bool Whether the model is trained in teacher_forcing mode. Note 1: In test mode, teacher_forcing should be set as False. Note 2: If teacher_forcing mode is on, # of frames in inputs = total_steps If teacher_forcing mode is off, # of frames in inputs = input_frames scheduled_sampling_ratio: float between [0, 1] The ratio of ground-truth frames used in teacher_forcing mode. default: 0 (i.e. no teacher forcing effectively) Returns: -------- outputs: a 5-th order tensor of size [batch_size, output_frames, hidden_channels, height, width] Output frames of the convolutional-LSTM module. """ # compute the teacher forcing mask if teacher_forcing and scheduled_sampling_ratio > 1e-6: # generate the teacher_forcing mask (4-th order) teacher_forcing_mask = torch.bernoulli( scheduled_sampling_ratio * torch.ones(inputs.size(0), future_frames - 1, 1, 1, 1, device=inputs.device)) else: # if not teacher_forcing or scheduled_sampling_ratio < 1e-6: teacher_forcing = False # the number of time steps in the computational graph total_steps = input_frames + future_frames - 1 outputs = [None] * total_steps for t in range(total_steps): # input_: 4-th order tensor of size [batch_size, input_channels, height, width] if t < input_frames: input_ = inputs[:, t] elif not teacher_forcing: input_ = outputs[t - 1] else: # if t >= input_frames and teacher_forcing: mask = teacher_forcing_mask[:, t - input_frames] input_ = inputs[:, t] * mask + outputs[t - 1] * (1 - mask) first_step = (t == 0) queue = [] # previous outputs for skip connection for b in range(self.num_blocks): for l in range(self.layers_per_block[b]): lid = "b{}l{}".format(b, l) # layer ID input_ = self.layers[lid](input_, first_step=first_step) queue.append(input_) if b >= self.skip_stride: input_ = torch.cat([input_, queue.pop(0)], dim=1) # concat over the channels # map the hidden states to predictive frames (with optional sigmoid function) outputs[t] = self.layers["output"](input_) if self.output_sigmoid: outputs[t] = torch.sigmoid(outputs[t]) # return the last output_frames of the outputs outputs = outputs[-output_frames:] # 5-th order tensor of size [batch_size, output_frames, channels, height, width] outputs = torch.stack([outputs[t] for t in range(output_frames)], dim=1) outputs = self.final_layer(outputs) return outputs[:, :, :, 0]
def eval_conditional_density( density: Any, condition: Tensor, limits: Tensor, dim1: int, dim2: int, resolution: int = 50, eps_margins1: Union[Tensor, float] = 1e-32, eps_margins2: Union[Tensor, float] = 1e-32, ) -> Tensor: r""" Return the unnormalized conditional along `dim1, dim2` given parameters `condition`. We compute the unnormalized conditional by evaluating the joint distribution: $p(x1 | x2) = p(x1, x2) / p(x2) \propto p(x1, x2)$ Args: density: Probability density function with `.log_prob()` method. condition: Parameter set that all dimensions other than dim1 and dim2 will be fixed to. Should be of shape (1, dim_theta), i.e. it could e.g. be a sample from the posterior distribution. The entries at `dim1` and `dim2` will be ignored. limits: Bounds within which to evaluate the density. Shape (dim_theta, 2). dim1: First dimension along which to evaluate the conditional. dim2: Second dimension along which to evaluate the conditional. resolution: Resolution of the grid along which the conditional density is evaluated. eps_margins1: We will evaluate the posterior along `dim1` at `limits[0]+eps_margins` until `limits[1]-eps_margins`. This avoids evaluations potentially exactly at the prior bounds. eps_margins2: We will evaluate the posterior along `dim2` at `limits[0]+eps_margins` until `limits[1]-eps_margins`. This avoids evaluations potentially exactly at the prior bounds. Returns: Conditional probabilities. If `dim1 == dim2`, this will have shape (resolution). If `dim1 != dim2`, it will have shape (resolution, resolution). """ condition = ensure_theta_batched(condition) theta_grid_dim1 = torch.linspace( float(limits[dim1, 0] + eps_margins1), float(limits[dim1, 1] - eps_margins1), resolution, ) theta_grid_dim2 = torch.linspace( float(limits[dim2, 0] + eps_margins2), float(limits[dim2, 1] - eps_margins2), resolution, ) if dim1 == dim2: repeated_condition = condition.repeat(resolution, 1) repeated_condition[:, dim1] = theta_grid_dim1 log_probs_on_grid = density.log_prob(repeated_condition) else: repeated_condition = condition.repeat(resolution**2, 1) repeated_condition[:, dim1] = theta_grid_dim1.repeat(resolution) repeated_condition[:, dim2] = torch.repeat_interleave( theta_grid_dim2, resolution) log_probs_on_grid = density.log_prob(repeated_condition) log_probs_on_grid = torch.reshape(log_probs_on_grid, (resolution, resolution)) # Subtract maximum for numerical stability. return torch.exp(log_probs_on_grid - torch.max(log_probs_on_grid))
transition = agent.step() buffer.add_to_buffer(transition) #Only start training after 100 transitions in the buffer mini_batch = buffer.sample_minbatch() loss = dqn.train_q_network(mini_batch) losses.append(loss) #append final loss for that episode training_iteration += 1 print("Iteration:", training_iteration) iterations.append(training_iteration) # GET FINAL Q VALUES FOR EACH STATE (EXTRACT FINAL LAYER FROM NETWORK, WITH EXISTING WEIGHTS I THINK?) #PROBABLY THIS RESHAPE ISN'T CORRECT, BUT SOMEHOW NEED TO WORK OUT HOW TO CORRECTLY RESHAPE FROM 4X100 TO 10X10X4. q_values = torch.reshape((dqn.q_network.output_layer.weight.data).t(), (10, 10, 4)).numpy() print(q_values.shape) # Draw final Q pattern visualiser = QValueVisualiser( environment=environment, magnification=500) #not sure if this is working correctly or not??? # Draw the image visualiser.draw_q_values( q_values ) #expecting q_values = np.random.uniform(0, 1, [10, 10, 4]) #Draw the ideal path (the greedy policy) found from training. #Find the greedy policy: Given a starting state, what is the max Q action, then move to that state and take next max Q to move there until end of episode. dqn.q_network.eval() #put into eval mode so it doesn't keep training. agent.reset() transitions = []
def forward(self, x): x = x.to(device) weights = self.hyper_net(x) weights = torch.reshape(weights, [self.output_size, -1]) return weights
from options.train_options import TrainOptions from utils.timer import Timer import os from data import CreateSrcDataLoader from data import CreateTrgDataLoader from model import CreateModel #import tensorboardX import torch.backends.cudnn as cudnn import torch from torch.autograd import Variable from utils import FDA_source_to_target import scipy.io as sio IMG_MEAN = np.array((104.00698793, 116.66876762, 122.67891434), dtype=np.float32) IMG_MEAN = torch.reshape(torch.from_numpy(IMG_MEAN), (1, 3, 1, 1)) CS_weights = np.array((1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0), dtype=np.float32) CS_weights = torch.from_numpy(CS_weights) def main(): opt = TrainOptions() args = opt.initialize() os.environ["CUDA_VISIBLE_DEVICES"] = args.GPU _t = {'iter time': Timer()} model_name = args.source + '_to_' + args.target if not os.path.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir)
def computeSquareEuclideanCostMatrix(self, Xs_batch, Xt_batch): return torch.reshape(torch.sum(torch.mul(Xs_batch, Xs_batch), dim=1), (-1, 1)) + torch.reshape(torch.sum(torch.mul(Xt_batch, Xt_batch), dim=1), (1, -1)) \ - 2. * torch.matmul(Xs_batch, torch.transpose(Xt_batch, 0,1))
def forward(self, z): z = self.linear(z) z = torch.reshape(z, (-1, 1, int(self.outsize/8), int(self.outsize/8))) loc_img = self.net(z) return loc_img
def reshape(input, shape): return th.reshape(input, shape)
def create_dense_flows(flattened_flows, batch_size, image_height, image_width): # possibly .view return torch.reshape(flattened_flows, [batch_size, image_height, image_width, 2])