def test_true_divide_out(self, device, dtype): dividend = (torch.randn(5, device=device) * 100).to(dtype) divisor = torch.arange(1, 6, device=device).to(dtype) # Tests that requests for an integer quotient fail if not dtype.is_floating_point: integral_quotient = torch.empty(5, device=device, dtype=dtype) with self.assertRaises(RuntimeError): torch.true_divide(dividend, divisor, out=integral_quotient) with self.assertRaises(RuntimeError): torch.true_divide(dividend, 2, out=integral_quotient) else: # Tests that requests for a floating quotient succeed floating_quotient = torch.empty(5, device=device, dtype=dtype) div_result = dividend / divisor self.assertEqual( div_result, torch.true_divide(dividend, divisor, out=floating_quotient)) self.assertEqual( dividend / 2, torch.true_divide(dividend, 2, out=floating_quotient))
torch.add(x, y, out=z1) print(z1) print(z1.shape) z2 = torch.add(x, y) print(z2) z3 = x + y print(z3) # subtraction z4 = x - y print(z4) # division z5 = torch.true_divide(x, y) print(z5) # multiplication z = x * y print(z) # inplace operations t = torch.zeros(3) t.add_(x) print(t) # exponentiation z = x.pow(2) print(z)
def decode(self, dec_state): """ decode """ long_tensor_type = torch.cuda.LongTensor if self.use_gpu else torch.LongTensor b = dec_state.get_batch_size() # [[0], [k*1], [k*2], ..., [k*(b-1)]] self.pos_index = (long_tensor_type(range(b)) * self.k).view(-1, 1) # Inflate the initial hidden states to be of size: (b*k, H) dec_state = dec_state.inflate(self.k) # Initialize the scores; for the first step, # ignore the inflated copies to avoid duplicate entries in the top k sequence_scores = long_tensor_type(b * self.k).float() sequence_scores.fill_(-float('inf')) sequence_scores.index_fill_( 0, long_tensor_type([i * self.k for i in range(b)]), 0.0) # Initialize the input vector input_var = long_tensor_type([self.BOS] * b * self.k) # Store decisions for backtracking stored_scores = list() stored_predecessors = list() stored_emitted_symbols = list() for t in range(1, self.max_length + 1): # Run the RNN one step forward output, dec_state, attn = self.model.decode(input_var, dec_state) log_softmax_output = output.squeeze(1) # To get the full sequence scores for the new candidates, add the # local scores for t_i to the predecessor scores for t_(i-1) sequence_scores = sequence_scores.unsqueeze(1).repeat(1, self.V) if self.length_average and t > 1: sequence_scores = sequence_scores * \ (1 - 1/t) + log_softmax_output / t else: sequence_scores += log_softmax_output scores, candidates = sequence_scores.view(b, -1).topk(self.k, dim=1) # Reshape input = (b*k, 1) and sequence_scores = (b*k) input_var = (candidates % self.V) sequence_scores = scores.view(b * self.k) input_var = input_var.view(b * self.k) # Update fields for next timestep if torch.__version__ == '1.2.0': predecessors = (candidates / self.V + self.pos_index.expand_as(candidates)).view( b * self.k) else: predecessors = (torch.true_divide(candidates, self.V) + self.pos_index.expand_as(candidates)).view( b * self.k).long() dec_state = dec_state.index_select(predecessors) # Update sequence scores and erase scores for end-of-sentence symbol so that they aren't expanded stored_scores.append(sequence_scores.clone()) eos_indices = input_var.data.eq(self.EOS) if eos_indices.nonzero(as_tuple=False).dim() > 0: sequence_scores.data.masked_fill_(eos_indices, -float('inf')) if self.ignore_unk: # Erase scores for UNK symbol so that they aren't expanded unk_indices = input_var.data.eq(self.UNK) if unk_indices.nonzero(as_tuple=False).dim() > 0: sequence_scores.data.masked_fill_(unk_indices, -float('inf')) # Cache results for backtracking stored_predecessors.append(predecessors) stored_emitted_symbols.append(input_var) predicts, scores, lengths = self._backtrack(stored_predecessors, stored_emitted_symbols, stored_scores, b) predicts = predicts[:, :1] scores = scores[:, :1] lengths = long_tensor_type(lengths)[:, :1] mask = sequence_mask(lengths, max_len=self.max_length).eq(0) predicts[mask] = self.PAD return predicts, lengths, scores
#====================================================================# # Tensor Math and Comparison Operations # #====================================================================# x = torch.tensor([1,2,3]) y = torch.tensor([9,8,7]) # Addition z1 = torch.add(x,y) z2 = x+y # Subtraction z = x-y # Division z = torch.true_divide(x,y) # element-wise division if equal size # inplace operations t = torch.zeros(3) t.add_(x) t += x # Exponentiation z = x.pow(2) z = x**2 # Simple comparison z = x > 0 print(z) # Matrix multiplication
def softplus(x, sigma=1.): y = torch.true_divide(x, sigma) z = x.clone().float() z[y < 34.0] = sigma * torch.log1p(torch.exp(y[y < 34.0])) return z
def forward(self, grads, *args, **kwargs): intrest = self.intrest_func(grads) intrest_sum = intrest.abs().sum(dim=[1, 2, 3]) total = grads.abs().sum(dim=[1, 2, 3]) return torch.true_divide(intrest_sum, total)
# Averaging models w_avg = copy.deepcopy(models[idx_user].state_dict()) # Record how many clients participated in this round of averaging. n_participants = 1 for i in range(args.num_users): if v_new[i] > v_old[i]: version_matrix[idx_user, i] = v_new[i] n_participants = n_participants + 1 w_model_to_merge = copy.deepcopy(models[i].state_dict()) n = len(w_avg.keys()) nt = 0 for key in w_avg.keys(): nt += 1 w_avg[key] = additive(w_avg[key], w_model_to_merge[key], args) for key in w_avg.keys(): w_avg[key] = torch.true_divide(w_avg[key], n_participants) print("Select user:"******", total number of participants:", n_participants, ", process:", r + 1, "/", args.num_users) global_model.load_state_dict(w_avg) # Update local model versions for selected clients version_matrix[idx_user, idx_user] = version_matrix[idx_user, idx_user] + 1 models[idx_user].load_state_dict(w_avg) local_model = LocalUpdate(args=args, dataset=train_dataset, idxs=user_groups[idx_user], logger=logger) w, loss, t_model = local_model.update_weights( model=copy.deepcopy(models[idx_user]), global_round=epoch) # print global training loss every 'print_every' round if (epoch + 1) % print_every == 0: local_weights = [] for i in range(args.num_users):
def test_sparse_true_divide(self, device, dtype): dividend = torch.randn(5, device=device).to(dtype) divisor = 2 dividend_sparse = dividend.to_sparse() casting_result = dividend.to(torch.get_default_dtype()) / 2 self.assertEqual(casting_result, torch.true_divide(dividend_sparse, 2).to_dense())
def error_loss(outputs, labels, mean_y, std_y): loss = torch.mean( torch.true_divide( torch.abs((outputs * std_y + mean_y) - (labels * std_y + mean_y)), (labels * std_y + mean_y))) return loss
for dp in raw_eval: eval_data.append([layer_structure + dp[0][0] + [dp[0][1]], dp[1][0]]) eval_data = np.array(eval_data) eval_inputs = torch.tensor( np.array([np.array(i) for i in eval_data[:, 0]], dtype='float32')) eval_targets = torch.tensor( np.array(eval_data[:, 1], dtype='float32').reshape(eval_data.shape[0], 1)) net.load_state_dict(torch.load(ckpt_path)) net.eval() eval_outputs = net(eval_inputs) print(eval_outputs.data.numpy()) print(eval_targets) print( torch.mean( torch.true_divide(torch.abs(eval_outputs - eval_targets), eval_targets)).item()) exit() pbar = tqdm(range(total_epochs)) border = "=" * 50 clear_border = _term_move_up() + "\r" + " " * len(border) + "\r" for epoch in pbar: # loop over the dataset multiple times running_loss = 0.0 for i, data in enumerate(train_loader, 0): # get the inputs; data is a list of [inputs, labels] inputs, labels = data inputs = Variable(inputs, volatile=True).cuda() labels = Variable(labels, volatile=True).cuda()
def compute(self): """ Aggregates and then calculates logs classification report similar to sklearn.metrics.classification_report. Typically used during epoch_end. Return: aggregated precision, recall, f1, report """ total_examples = torch.sum(self.num_examples_per_class) num_non_empty_classes = torch.nonzero( self.num_examples_per_class).size(0) precision = torch.true_divide(self.tp * 100, (self.tp + self.fp + METRIC_EPS)) recall = torch.true_divide(self.tp * 100, (self.tp + self.fn + METRIC_EPS)) f1 = torch.true_divide(2 * precision * recall, (precision + recall + METRIC_EPS)) report = '\n{:50s} {:10s} {:10s} {:10s} {:10s}'.format( 'label', 'precision', 'recall', 'f1', 'support') for i in range(len(self.tp)): label = f'label_id: {i}' if self.ids_to_labels and i in self.ids_to_labels: label = f'{self.ids_to_labels[i]} ({label})' report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( label, precision[i], recall[i], f1[i], self.num_examples_per_class[i]) micro_precision = torch.true_divide( torch.sum(self.tp) * 100, torch.sum(self.tp + self.fp) + METRIC_EPS) micro_recall = torch.true_divide( torch.sum(self.tp) * 100, torch.sum(self.tp + self.fn) + METRIC_EPS) micro_f1 = torch.true_divide( 2 * micro_precision * micro_recall, (micro_precision + micro_recall + METRIC_EPS)) macro_precision = torch.sum(precision) / num_non_empty_classes macro_recall = torch.sum(recall) / num_non_empty_classes macro_f1 = torch.sum(f1) / num_non_empty_classes weighted_precision = torch.sum( precision * self.num_examples_per_class) / total_examples weighted_recall = torch.sum( recall * self.num_examples_per_class) / total_examples weighted_f1 = torch.sum( f1 * self.num_examples_per_class) / total_examples report += "\n-------------------" report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( 'micro avg', micro_precision, micro_recall, micro_f1, total_examples) report += '\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( 'macro avg', macro_precision, macro_recall, macro_f1, total_examples) report += ('\n{:50s} {:8.2f} {:8.2f} {:8.2f} {:8.0f}'.format( 'weighted avg', weighted_precision, weighted_recall, weighted_f1, total_examples) + '\n') self.total_examples = total_examples if self.mode == 'macro': return macro_precision, macro_recall, macro_f1, report elif self.mode == 'weighted': return weighted_precision, weighted_recall, weighted_f1, report elif self.mode == 'micro': return micro_precision, micro_recall, micro_f1, report elif self.mode == 'all': return precision, recall, f1, report else: raise ValueError( f'{self.mode} mode is not supported. Choose "macro" to get aggregated numbers \ or "all" to get values for each class.')
def __call__(self): for epoch in range(10000): total_loss = 0 for i, (img_data, label, box, landmarks) in enumerate(self.train_dataloader): if torch.cuda.is_available() is True: img_data = img_data.cuda() gt_label = label.cuda() gt_boxes = box.cuda() gt_landmarks = landmarks.cuda() else: gt_label = label gt_boxes = box gt_landmarks = landmarks pred_label, pred_offset, pred_landmarks = self.net(img_data) pred_label = pred_label.view(-1, 2) pred_offset = pred_offset.view(-1, 4) pred_landmarks = pred_landmarks.view(-1, 10) # print(pred_label.shape, pred_offset.shape, pred_landmarks.shape) # print(label.shape, box.shape, landmarks.shape) self.opt.zero_grad() cls_loss = self.cls_loss(gt_label, pred_label) box_loss = self.box_loss(gt_label, gt_boxes, pred_offset) landmark_loss = self.landmark_loss(gt_label, gt_landmarks, pred_landmarks) loss = cls_loss + box_loss + landmark_loss loss.backward() self.opt.step() total_loss += loss.cpu().detach() # self.summarywrite.add_scalars('train/loss', loss.cpu().detach().item(), global_step=self.epoch_num) self.summarywrite.add_scalars("train/loss", {i: j for i, j in zip(["loss", "cls_loss", "box_loss", "landmark_loss"], [loss.cpu().detach().item(), cls_loss.cpu().item(), box_loss.cpu().item(), landmark_loss.cpu().item()]) }, global_step=self.global_step) self.global_step += 1 print( f"epoch:{self.epoch_num}---loss:{loss.cpu().item()}---cls_loss:{cls_loss.cpu().item()}---box_loss:{box_loss.cpu().item()}---landmark_loss:{landmark_loss.cpu().item()}") self.save_state_dict() self.export_model(f"./param/{self.net_stage}.pt") if self.epoch_num % 10 == 0: with torch.no_grad(): for name, parmeter in self.net.named_parameters(): if parmeter.grad is not None: avg_grad = torch.mean(parmeter.grad) print(f"{name}----grad_avg:{avg_grad}") self.summarywrite.add_scalar(f"grad_avg/{name}", avg_grad.item(), self.epoch_num) self.summarywrite.add_histogram(f"grad/{name}", parmeter.cpu().numpy(), self.epoch_num) if parmeter.data is not None: avg_weight = torch.mean(parmeter.data) print(f"{name}----weight_avg:{avg_weight}") self.summarywrite.add_scalar(f"weight_avg/{name}", avg_weight.item(), self.epoch_num) self.summarywrite.add_histogram(f"weight/{name}", parmeter.cpu().numpy(), self.epoch_num) total = 0 right = 0 tp = 0 fp = 0 fn = 0 tn = 0 total_cls_loss = 0 total_box_loss = 0 total_landmark_loss = 0 for i, (img_data, label, box, landmarks) in enumerate(self.eval_dataloader): if torch.cuda.is_available() is True: img_data = img_data.cuda() gt_label = label.cuda() gt_boxes = box.cuda() gt_landmarks = landmarks.cuda() else: gt_label = label gt_boxes = box gt_landmarks = landmarks with torch.no_grad(): pred_label, pred_offset, pred_landmarks = self.net(img_data) print(pred_label, pred_offset, pred_landmarks) pred_label = pred_label.view(-1, 2) pred_offset = pred_offset.view(-1, 4) pred_landmarks = pred_landmarks.view(-1, 10) total_cls_loss += self.cls_loss(gt_label, pred_label) total_box_loss += self.box_loss(gt_label, gt_boxes, pred_offset) total_landmark_loss += self.landmark_loss(gt_label, gt_landmarks, pred_landmarks) pred_label = torch.argmax(pred_label, dim=1) mask = gt_label <= 1 right += torch.sum(gt_label[mask] == pred_label[mask]) total += gt_label[mask].shape[0] p_mask = gt_label == 1 tp += torch.sum(gt_label[p_mask] == pred_label[p_mask]) fp += torch.sum(gt_label[p_mask] != pred_label[p_mask]) n_mask = gt_label == 0 tn += torch.sum(gt_label[n_mask] == pred_label[n_mask]) fn += torch.sum(gt_label[n_mask] != pred_label[n_mask]) # acc = right.cpu().detach() / total acc = torch.true_divide(right, total) # precision = tp / (tp + fp) precision = torch.true_divide(tp, (tp + fp)) # recall = tp / (tp + fn) recall = torch.true_divide(tp, (tp + fn)) # f1 = 2 * precision * recall / (precision + recall) f1 = torch.true_divide((2 * precision * recall), (precision + recall)) avg_cls_loss = total_cls_loss / i avg_box_loss = total_box_loss / i avg_lanmark_loss = total_landmark_loss / i self.summarywrite.add_scalars("eval/loss", {i: j for i, j in zip(["avg_cls_loss", "avg_box_loss", "avg_lanmark_loss"], [avg_cls_loss.cpu().item(), avg_box_loss.cpu().item(), avg_lanmark_loss.cpu().item()]) }, global_step=self.epoch_num) self.summarywrite.add_scalars("eval_set", { i: j for i, j in zip(["acc", "precision", "recall", "f1"], [acc, precision, recall, f1]) }, global_step=self.epoch_num) print("Epoch %d, " % self.epoch_num, f"result on eval set: acc {acc.cpu().item()}", f"precision {precision.cpu().item()}", f"recall {recall.cpu().item()}", f"f1 {f1.cpu().item()}") self.epoch_num += 1
colorimg[y, x, :] = colors[2] elif index == 1: colorimg[y, x, :] = colors[1] elif index == 3: colorimg[y, x, :] = colors[3] else: colorimg[y, x, :] = colors[0] # colorimg[y,x,:] = np.mean(selected_colors, axis=0) return colorimg.astype(np.uint8) for idx, data in enumerate(val_loader): image, mask = data input = torch.true_divide(image, 255).to(torch.float).to(device) target = mask.to(torch.long).to(device) output = model(input) output = softmax(output).squeeze(dim=0).detach().cpu() rgb_out = masks_to_colorimg(output.numpy()) os.mkdir(f'{OUTPUT_FOLDER}/sample_{idx}') io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/INPUT.png', image.squeeze(0).permute(1, 2, 0).numpy()) io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/GT.png', masks_to_colorimg(mask.squeeze(0).numpy())) io.imsave(f'{OUTPUT_FOLDER}/sample_{idx}/pred.png', rgb_out) if idx > 4: break
two_pi = torch.tensor(2 * np.pi, dtype=torch.float).cuda() tVec = torch.arange(0, sequenceDuration, 1 / fs, dtype=torch.float).cuda() chripDurations = (minChirpDuration + torch.mul(torch.rand(batch_size, dtype=torch.float), maxChirpDuration - minChirpDuration)).cuda() # sec startFreqs = torch.mul(torch.rand(batch_size, dtype=torch.float), fs / 2).cuda() # [hz/sec] startPhases = torch.mul(torch.rand(batch_size, dtype=torch.float), two_pi).cuda() # [rad/sec] k = torch.pow( 4000 / 100, 1 / chripDurations ) # k is the rate of exponential change in frequency to transform from 100 to 4000 hz in 10 seconds freqFactor = torch.true_divide( torch.pow(k[None, :].repeat(tVec.shape[0], 1), tVec[:, None].repeat( 1, k.shape[0])) - 1, torch.log(k)[None, :].repeat(tVec.shape[0], 1)) phases = torch.mul( torch.mul(startFreqs[None, :].repeat(tVec.shape[0], 1), two_pi), freqFactor) + startPhases[None, :].repeat(tVec.shape[0], 1) pureSinWaves = torch.sin(phases) noise = torch.mul(torch.randn_like(pureSinWaves), noiseStd) noisySinWaves = pureSinWaves + noise modelInputMean_dbW = 10 * np.log10( np.mean(np.power(noisySinWaves.cpu().numpy(), 2))) ''' plt.plot(tVec.cpu().numpy(), noisySinWaves[:, 0].cpu().numpy()) plt.xlabel('sec') plt.title('LSTM input wave')
def train_net_fednova(net_id, net, global_model, train_dataloader, test_dataloader, epochs, lr, args_optimizer, device="cpu"): logger.info('Training network %s' % str(net_id)) train_acc = compute_accuracy(net, train_dataloader, device=device) test_acc, conf_matrix = compute_accuracy(net, test_dataloader, get_confusion_matrix=True, device=device) logger.info('>> Pre-Training Training accuracy: {}'.format(train_acc)) logger.info('>> Pre-Training Test accuracy: {}'.format(test_acc)) optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=lr, momentum=args.rho, weight_decay=args.reg) criterion = nn.CrossEntropyLoss().to(device) if type(train_dataloader) == type([1]): pass else: train_dataloader = [train_dataloader] #writer = SummaryWriter() tau = 0 for epoch in range(epochs): epoch_loss_collector = [] for tmp in train_dataloader: for batch_idx, (x, target) in enumerate(tmp): x, target = x.to(device), target.to(device) optimizer.zero_grad() x.requires_grad = True target.requires_grad = False target = target.long() out = net(x) loss = criterion(out, target) loss.backward() optimizer.step() tau = tau + 1 epoch_loss_collector.append(loss.item()) epoch_loss = sum(epoch_loss_collector) / len(epoch_loss_collector) logger.info('Epoch: %d Loss: %f' % (epoch, epoch_loss)) a_i = (tau - args.rho * (1 - pow(args.rho, tau)) / (1 - args.rho)) / (1 - args.rho) global_model_para = global_model.state_dict() net_para = net.state_dict() norm_grad = copy.deepcopy(global_model.state_dict()) for key in norm_grad: #norm_grad[key] = (global_model_para[key] - net_para[key]) / a_i norm_grad[key] = torch.true_divide( global_model_para[key] - net_para[key], a_i) train_acc = compute_accuracy(net, train_dataloader, device=device) test_acc, conf_matrix = compute_accuracy(net, test_dataloader, get_confusion_matrix=True, device=device) logger.info('>> Training accuracy: %f' % train_acc) logger.info('>> Test accuracy: %f' % test_acc) logger.info(' ** Training complete **') return train_acc, test_acc, a_i, norm_grad
def binary_dice(pred: Tensor, target: Tensor, eps=1e-9) -> torch.float: """NOTE: Must copy to CPU, so expensive. Should not be peformed every global step.""" p, t = pred.cpu().detach().view(-1), target.cpu().detach().view(-1) intersect = (t * p).sum() return torch.true_divide(2 * intersect, t.sum() + p.sum() + eps)
def prediction_loop(self, dataloader: DataLoader, description: str, prediction_loss_only: Optional[bool] = None, extract_path: Optional[str] = None, cache_path: Optional[str] = None) -> PredictionOutput: """ Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`. Works both with or without labels. """ prediction_loss_only = (prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only) model = self.model # multi-gpu eval if self.args.n_gpu > 1: model = torch.nn.DataParallel(model) else: model = self.model # Note: in torch.distributed mode, there's no point in wrapping the model # inside a DistributedDataParallel as we'll be under `no_grad` anyways. batch_size = dataloader.batch_size eval_losses: List[float] = [] hidden_states: torch.tensor = None preds: torch.Tensor = None label_ids: torch.Tensor = None model.eval() if self.args.past_index >= 0: self._past = None # Unfortunate, but we'll run through the dataloader once to count the number of tokens (or this could be pre-processed) if extract_path is not None: stimulus_mask = lambda tokens: (tokens != 101) & (tokens != 102 ) & (tokens != 0) cached_masks = None if osp.exists(f"{cache_path}.npy"): # np instead of torch, something's funky with Vivek's env. cached_masks = torch.from_numpy(np.load(f"{cache_path}.npy")) else: all_masks = None limit_tokens = self.custom_cfg.TASK.EXTRACT_TOKENS_LIMIT # Calculate the random ratio of tokens to grab (we specify number of tokens to extract) total_tokens = 0 for inputs in dataloader: tokens = inputs["input_ids"] total_tokens += stimulus_mask(tokens).sum() subset_ratio = torch.true_divide(limit_tokens, total_tokens) # Seed, we want to be sure that we're finding the same stimuli disable_tqdm = not self.is_local_process_zero( ) or self.args.disable_tqdm samples_count = 0 for inputs in tqdm(dataloader, desc=description, disable=disable_tqdm): loss, logits, labels, states = self.prediction_step( model, inputs, prediction_loss_only, output_hidden_states=extract_path is not None) batch_size = inputs[list(inputs.keys())[0]].shape[0] if loss is not None: eval_losses.append(loss * batch_size) if states is not None: # L + 1 [ Batch x Length x Hidden ] (layers and embedding) if cached_masks is not None: cached_masks = cached_masks.to(logits.device) mask = cached_masks[samples_count:samples_count + inputs["input_ids"].shape[0]] # B x T mask = mask[:, :inputs["input_ids"]. shape[1]] # Dynamic padding else: subset_mask = torch.full(inputs["input_ids"].shape, subset_ratio, device=logits.device) mask = (torch.bernoulli(subset_mask).long() & stimulus_mask(inputs["input_ids"])).bool() # B X T if all_masks is None: all_masks = mask else: all_masks = nested_concat(all_masks, mask, padding_index=-100) # B x T # [1:] to drop embedding layer states = torch.stack(states)[1:].permute(1, 2, 0, 3) # B x T x L x H target_tokens = states[mask] # M x L x H if hidden_states is None: hidden_states = target_tokens else: hidden_states = torch.cat([hidden_states, target_tokens], dim=0) samples_count += batch_size if logits is not None: preds = logits if preds is None else nested_concat( preds, logits, padding_index=-100) if labels is not None: label_ids = labels if label_ids is None else nested_concat( label_ids, labels, padding_index=-100) if extract_path is not None: os.makedirs(osp.split(extract_path)[0], exist_ok=True) np.save(extract_path, hidden_states.half().cpu().numpy()) # half to save memory if cached_masks is None: os.makedirs(osp.split(cache_path)[0], exist_ok=True) np.save(cache_path, all_masks.cpu().numpy()) if self.args.past_index and hasattr(self, "_past"): # Clean the state at the end of the evaluation loop delattr(self, "_past") if self.args.local_rank != -1: # In distributed mode, concatenate all results from all nodes: if preds is not None: preds = self.distributed_concat( preds, num_total_examples=self.num_examples(dataloader)) if label_ids is not None: label_ids = self.distributed_concat( label_ids, num_total_examples=self.num_examples(dataloader)) # Finally, turn the aggregated tensors into numpy arrays. if preds is not None: preds = preds.cpu().numpy() if label_ids is not None: label_ids = label_ids.cpu().numpy() if self.compute_metrics is not None and preds is not None and label_ids is not None: metrics = self.compute_metrics( EvalPrediction(predictions=preds, label_ids=label_ids)) else: metrics = {} if len(eval_losses) > 0: metrics["eval_loss"] = np.sum(eval_losses) / samples_count # Prefix all keys with eval_ for key in list(metrics.keys()): if not key.startswith("eval_"): metrics[f"eval_{key}"] = metrics.pop(key) return PredictionOutput(predictions=preds, label_ids=label_ids, metrics=metrics)
def test_true_divide(self, device, dtype): dividend = torch.randn(5, device=device).to(dtype) divisor = torch.arange(1, 6, device=device).to(dtype) casting_result = dividend.to(torch.get_default_dtype()) / divisor.to( torch.get_default_dtype()) self.assertEqual(casting_result, torch.true_divide(dividend, divisor))
def _true_divide(dividend, divisor): return torch.true_divide(dividend, divisor)
def forward(self, x, y): return torch.true_divide(x, y)
def train_model(model, optimizer, scheduler, num_epochs=20): best_model_wts = copy.deepcopy(model.state_dict()) best_loss = 1e10 patience = 0 for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) since = time.time() # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': scheduler.step() for param_group in optimizer.param_groups: print("LR", param_group['lr']) model.train() # Set model to training mode else: model.eval() # Set model to evaluate mode metrics = defaultdict(float) epoch_samples = 0 for inputs, labels in dataloaders[phase]: inputs = torch.true_divide(inputs, 255) inputs = inputs.type(torch.float) labels = labels.type(torch.float) inputs = inputs.to(device) labels = labels.to(device) # zero the parameter gradients optimizer.zero_grad() # forward # track history if only in train with torch.set_grad_enabled(phase == 'train'): outputs = model(inputs) loss = calc_loss(outputs, labels, metrics) # backward + optimize only if in training phase if phase == 'train': loss.backward() optimizer.step() # statistics epoch_samples += inputs.size(0) print_metrics(metrics, epoch_samples, phase) epoch_loss = metrics['loss'] / epoch_samples # deep copy the model if phase == 'val' and epoch_loss < best_loss: print("saving best model") best_loss = epoch_loss best_model_wts = copy.deepcopy(model.state_dict()) patience = 0 elif phase == 'val': patience += 1 time_elapsed = time.time() - since print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60)) if patience >= 15: print("out of patience breaking the training loop :(") break print('Best val loss: {:4f}'.format(best_loss)) # load best model weights model.load_state_dict(best_model_wts) return model
def forward(self, x, y): # Add transpose to hide shape/type information # Otherwise shape and type are still avaiable from input. x = x.transpose(1, 2) y = y.transpose(1, 2) return torch.true_divide(x, y)
def nnet(train_set, test_set, train_labels, test_labels, classes): # 1. Shuffle train and test set train_set = np.transpose(train_set, (0, 3, 1, 2)) test_set = np.transpose(test_set, (0, 3, 1, 2)) train_set = train_set.astype('float32') test_set = test_set.astype('float32') #print('Dataset loaded.') indices = np.arange(train_set.shape[0]) np.random.shuffle(indices) train_set = train_set[indices] train_labels = train_labels[indices] indices = np.arange(test_set.shape[0]) np.random.shuffle(indices) test_set = test_set[indices] test_labels = test_labels[indices] # 2. Transform train and test set into tensors train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=False) test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False) train_labels = torch.tensor(train_labels, dtype=torch.long) test_labels = torch.tensor(test_labels, dtype=torch.long) # Use torch.true_divide(image, 255) insted #transform = tr.Normalize((128, 128, 128), (127, 127, 127)) # 3. Create a model net = Net() # 4. Define a loss function and the optimizer criterion = nn.CrossEntropyLoss() # or MSELoss optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9) #optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate) # 5. Train the network (if necessary) net.train() if os.path.isfile('.pth'): net.load_state_dict(torch.load('.pth')) else: for epoch in range(epochs): # loop over the dataset multiple times running_loss = 0.0 for i, inputs in enumerate(train_loader): curr_size = inputs.size()[0] for j in range(curr_size): inputs[j] = torch.true_divide(inputs[j], 255) outputs = net(inputs) #right_prob = torch.zeros([curr_size, n_classes], dtype=torch.float32) #for j in range(curr_size): # right_prob[j, int(train_labels[i*batch_size+j])] = 1 loss = criterion( outputs, train_labels[i * batch_size:(i + 1) * batch_size]) loss.backward() optimizer.step() optimizer.zero_grad() # Print statistics running_loss += loss.item() if i % 100 == 0: print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 100)) running_loss = 0.0 print('Training ended.') # Save the obtained model torch.save(net.state_dict(), 'trained_network.pth') # 6. Test the network net.eval() # disable batch normalization cmc = np.zeros((n_classes, n_classes)) correct = 0 total = 0 total_predicted = [] with torch.no_grad(): for i, inputs in enumerate(test_loader): curr_size = inputs.size()[0] for j in range(curr_size): inputs[j] = torch.true_divide(inputs[j], 255) outputs = net(inputs) predicted = torch.argmax(outputs, dim=1) total_predicted.extend(predicted) total += curr_size correct += (predicted == test_labels[i * batch_size:(i + 1) * batch_size]).sum().item() for predict, test_label in zip( predicted, test_labels[i * batch_size:(i + 1) * batch_size]): cmc[int(test_label), int(predict)] += 1.0 #if i % 10 == 0: #print(predicted) #print(test_labels[i*batch_size:(i+1)*batch_size], end='\n') precision = [] recall = [] for i in range(n_classes): if cmc[i, i] != 0: precision.append(cmc[i, i] / np.sum(cmc[:, i])) recall.append(cmc[i, i] / np.sum(cmc[i, :])) precision = np.mean(np.asarray(precision)) recall = np.mean(np.asarray(recall)) print('Accuracy of the network on the test images: {0:.2f} %'.format( 100 * correct / total)) print('Classifier\'s mean precision: ' + "{0:.2f}".format(precision)) print('Classifier\'s mean recall: ' + "{0:.2f}".format(recall)) # 7. Test performance for every class class_correct = list(0. for i in range(n_classes)) class_total = list(0. for i in range(n_classes)) with torch.no_grad(): for i, inputs in enumerate(test_loader): curr_size = inputs.size()[0] for j in range(curr_size): inputs[j] = torch.true_divide(inputs[j], 255) outputs = net(inputs) predicted = torch.argmax(outputs, dim=1) c = (predicted == test_labels[i * batch_size:(i + 1) * batch_size]).squeeze() for j in range(curr_size): label = test_labels[i * batch_size + j] class_correct[int(label)] += c[j].item() class_total[int(label)] += 1 print() for i in range(n_classes): print('Accuracy of %5s: %2d %%' % (classes[i], 100 * class_correct[i] / class_total[i])) print() return total_predicted
def do_train(): mean, std, count = statistic(args.train_path) # print("Mean: {}, Standard deviation: {}, count: {}".format(mean, std, count)) custom_transform = transforms.Compose([ transforms.Resize([args.size, args.size]), transforms.ToTensor(), transforms.Normalize(mean, std) ]) total = sum(count) weight = [] for number in count: weight += [torch.true_divide(total, number)] * number # print("Check weight length: ", len(weight)) # print("Check total length: ", int(total)) sampler = torch.utils.data.sampler.WeightedRandomSampler( weights=torch.Tensor(weight), num_samples=int(total)) del count, weight train_set = ImageFolder(root=args.train_path, transform=custom_transform) val_set = ImageFolder(root=args.validation_path, transform=custom_transform) train_loader = DataLoader(train_set, batch_size=args.bs, \ pin_memory=True, drop_last=True, sampler=sampler) val_loader = DataLoader(val_set, batch_size=args.bs) if args.loadOrNot == 1: model = torch.load(args.model_path + args.model_load) else: model = Classifier().cuda() LossFunction = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=0.05) # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) # optimizer = torch.optim.SGD( # model.parameters(), lr=args.lr, momentum=0.9) BestAccuracy = 0.0 if args.model_save: torch.save(model, args.model_path + args.model_save) for epoch in range(args.ep): model.train() for i, data in enumerate(train_loader): optimizer.zero_grad() train_pred = model(data[0].cuda()) batch_loss = LossFunction(train_pred, data[1].cuda()) batch_loss.backward() optimizer.step() train_loss = 0.0 train_acc = 0.0 model.eval() for i, data in enumerate(train_loader): train_pred = model(data[0].cuda()) batch_loss = LossFunction(train_pred, data[1].cuda()) train_acc += np.sum( np.argmax(train_pred.cpu().data.numpy(), axis=1) == data[1].numpy()) train_loss += batch_loss.item() val_acc = 0.0 model.eval() for i, data in enumerate(val_loader): val_pred = model(data[0].cuda()) val_acc += np.sum( np.argmax(val_pred.cpu().data.numpy(), axis=1) == data[1].numpy()) accuracy = val_acc / len(val_set) print('Epoch: %d, Train Acc: %3.6f Loss: %3.6f Val Acc: %3.6f' % (epoch, train_acc / len(train_set), train_loss / len(train_set), accuracy)) if args.model_save and accuracy > BestAccuracy: BestAccuracy = accuracy torch.save(model, args.model_path + args.model_save)
def lif_j(j, tau_ref, tau_rc, amplitude=1.): j = torch.true_divide(1., j) j = torch.log1p(j) j = tau_ref + tau_rc * j j = torch.true_divide(amplitude, j) return j
loss.backward() optimizer.step() # GTLVQ uses projected SGD, which means to orthogonalize the subspaces after every gradient update. model.gtlvq.orthogonalize_subspace() if batch_idx % log_interval == 0: acc = calculate_prototype_accuracy(distances, y_train, plabels) print( f"Epoch: {epoch + 1:02d}/{num_epochs:02d} Epoch Progress: {100. * batch_idx / len(train_loader):02.02f} % Loss: {loss.item():02.02f} \ Train Acc: {acc.item():02.02f}") # Test with torch.no_grad(): model.eval() correct = 0 total = 0 for x_test, y_test in test_loader: x_test, y_test = x_test.to(device), y_test.to(device) test_distances = model(torch.tensor(x_test)) test_plabels = model.gtlvq.cls.prototype_labels.to(device) i = torch.argmin(test_distances, 1) correct += torch.sum(y_test == test_plabels[i]) total += y_test.size(0) print("Accuracy of the network on the test images: %d %%" % (torch.true_divide(correct, total) * 100)) # Save the model PATH = "./glvq_mnist_model.pth" torch.save(model.state_dict(), PATH)
def get_accuracy(logit, true_y): pred_y = torch.argmax(logit, dim=1) return torch.true_divide((pred_y == true_y).sum(), len(true_y))
def test_implementation_des(cfg): """ Simulates a train and val epoch to check if the gradients are being updated, metrics are being calculated correctly Args: cfg (CfgNode): configs. Details can be found in slowfast/config/defaults.py """ # Set random seed from configs. np.random.seed(cfg.RNG_SEED) torch.manual_seed(cfg.RNG_SEED) # Setup logging format. logging.setup_logging(cfg.OUTPUT_DIR) # Print config. logger.info("Test implementation") # Build the video model and print model statistics. model = build_clevrer_model(cfg) # Construct the optimizer. optimizer = optim.construct_optimizer(model, cfg) start_epoch = cu.load_train_checkpoint(cfg, model, optimizer) # Create the video train and val loaders. if cfg.TRAIN.DATASET != 'Clevrer_des': print("This train script does not support your dataset: -{}-. Only Clevrer_des".format(cfg.TRAIN.DATASET)) exit() train_loader = build_dataloader(cfg, "train") val_loader = build_dataloader(cfg, "val") # Create meters. train_meter = ClevrerTrainMeter(len(train_loader), cfg) val_meter = ClevrerValMeter(len(val_loader), cfg) # Perform the training loop. logger.info("Start epoch: {}".format(start_epoch + 1)) # Train for one epoch. model_before = copy.deepcopy(model) cur_epoch = start_epoch train_epoch( train_loader, model, optimizer, train_meter, cur_epoch, cfg, test_imp=True ) print("Check how much parameters changed") for (p_b_name, p_b), (p_name, p) in zip(model_before.named_parameters(), model.named_parameters()): if p.requires_grad: print("Parameter requires grad:") print(p_name, p_b_name) #Calculate ratio of change change = torch.abs(torch.norm(p) - torch.norm(p_b)) print("Ratio of change = {}".format(torch.true_divide(change, torch.norm(p_b)))) if (p_b != p).any(): print("--Check--") else: print("ALERT - WEIGHTS DID NOT CHANGE WITH TRAINING.") else: print("Parameter does not require grad:") print(p_name) print(p) print("Val epoch") eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, test_imp=True)
print(2) dataLoader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, num_workers=2, shuffle=True) optim = torch.optim.Adam(net.parameters(), lr=0.1, betas=(0.5, 0.999)) fixed_x, fixed_label = dataset.get_fixed() fixed_x = torch.tensor(fixed_x) fixed_label = torch.tensor(fixed_label) print(3) for epoch in range(1): for i, data in tqdm(enumerate(dataLoader, 0)): net.zero_grad() x = Variable(data[0]) label = Variable(data[1]) y = net(x) loss = torch.nn.BCELoss()(y, label) optim.zero_grad() loss.backward() optim.step() print('loss: ', loss.mean()) test_y = net(x) real = torch.argmax(y, dim=1) predict = torch.argmax(test_y, dim=1) acc = torch.true_divide(torch.sum(predict == real), len(real)) print('acc: ', acc)
val_lo = (val_total_loss / len(valiter)) val_acc = [val_accuracy[i] / len(valiter) for i in range(2)] print(f'Epoch {epoch}: train_loss: {train_lo:.4f} f1_macro: {train_acc[0]} f1_micro {train_acc[1]} | val_loss: {val_lo:.4f} f1_macro: {val_acc[0]} f1_micro {val_acc[1]}') return model if __name__ == '__main__': cuda = torch.cuda.is_available() device = torch.device("cpu") if not cuda else torch.device("cuda:0") dataL = dataload() text_field, label_field, trainds, valds, vectors = dataL.buildVocab() values = [label_field.vocab.freqs.most_common(15)[i][1] for i in range(15)] weight = torch.true_divide(torch.tensor(values), sum(values)) crit = nn.CrossEntropyLoss(weight=weight.cuda()) batch_size = 64 traindl, valdl = data.BucketIterator.splits(datasets=(trainds, valds), batch_size=batch_size, sort_key=lambda x: len(x.text), device=device, sort_within_batch=True, repeat=False) # print("[Corpus]: train: {}, test: {}, vocab: {}, labels: {}".format( # len(train_iter.dataset), len(val_iter.dataset), len(TEXT.vocab), len(LABEL.vocab))) train_iter, val_iter = traindl, valdl TEXT, LABEL = text_field, label_field ntokens, nlabels = len(TEXT.vocab), len(LABEL.vocab)