def run_infer(n_classes, from_tfr, in_memory_data=None, return_vals=False): graph = tf.get_default_graph() sess = tf.Session(graph=graph) if from_tfr: C_dataset = TFR_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, TEST_ROOT, 10000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, file_shards=FLAGS.train_shards) C_dataset.make_tfr_dataset() else: C_dataset = Memory_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, UP_TEST, 2000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES) C_dataset.make_memory_dataset() C_dataset.make_memory_feed_dict(in_memory_data) fluid_iterator = Dataset_iterator(C_dataset.dataset) el = fluid_iterator.get_el() net = Recognition_Network(n_classes, 3, 0.05, sess, graph, P_VALUE) net.set_params(el) net.build_optimizer(lr=0.01, resnet_path=None, patch_model_path=PATCH_MODEL, total_model_path=None) if from_tfr: sess.run(fluid_iterator.return_initialized_iterator(C_dataset.dataset)) else: sess.run(fluid_iterator.return_initialized_iterator(C_dataset.dataset), feed_dict=C_dataset.feed_dict) pred_dict, labels_list = create_holders(n_classes) while True: try: scores, singles = net.score() for i, j in scores.items(): pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0) labels_list = np.concatenate([labels_list, singles], axis=0) except tf.errors.OutOfRangeError: break print('The metrics are: ') eval_metrics(pred_dict, labels_list, plot_auc=False) if return_vals: return pred_dict, labels_list
def train(args, epoch): global LOSS_0 losses, psnrs, ssims, lpips = utils.init_meters(args.loss) model.train() criterion.train() t = time.time() for i, (images, imgpaths) in enumerate(train_loader): # Build input batch im1, im2, gt = utils.build_input(images, imgpaths) # Forward optimizer.zero_grad() out, feats = model(im1, im2) loss, loss_specific = criterion(out, gt, None, feats) # Save loss values for k, v in losses.items(): if k != 'total': v.update(loss_specific[k].item()) if LOSS_0 == 0: LOSS_0 = loss.data.item() losses['total'].update(loss.item()) # Backward (+ grad clip) - if loss explodes, skip current iteration loss.backward() if loss.data.item() > 10.0 * LOSS_0: print(max(p.grad.data.abs().max() for p in model.parameters())) continue torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1) optimizer.step() # Calc metrics & print logs if i % args.log_iter == 0: utils.eval_metrics(out, gt, psnrs, ssims, lpips, lpips_model) print('Train Epoch: {} [{}/{}]\tLoss: {:.6f}\tPSNR: {:.4f}\tTime({:.2f})'.format( epoch, i, len(train_loader), losses['total'].avg, psnrs.avg, time.time() - t)) # Log to TensorBoard utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg, optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i) # Reset metrics losses, psnrs, ssims, lpips = utils.init_meters(args.loss) t = time.time()
def eval_model(model, data): model.eval() reference, candidate, source, alignments = [], [], [], [] tgt_vocab = data['tgt_vocab'] count, total_count = 0, len(data['testset']) dataloader = data['testloader'] for src, tgt, src_len, tgt_len, original_src, original_tgt in dataloader: if config.use_cuda: src = src.cuda() src_len = src_len.cuda() with torch.no_grad(): if config.beam_size > 1 and (not config.global_emb): samples, alignment, _ = model.beam_sample( src, src_len, beam_size=config.beam_size, eval_=True) else: samples, alignment = model.sample(src, src_len) candidate += [ tgt_vocab.convertToLabels(s.tolist(), utils.EOS) for s in samples ] source += original_src reference += original_tgt if alignment is not None: alignments += [align for align in alignment] count += len(original_src) utils.progress_bar(count, total_count) if config.unk and config.attention != 'None': cands = [] for s, c, align in zip(source, candidate, alignments): cand = [] for word, idx in zip(c, align): if word == utils.UNK_WORD and idx < len(s): try: cand.append(s[idx]) except: cand.append(word) print("%d %d\n" % (len(s), idx)) else: cand.append(word) cands.append(cand) if len(cand) == 0: print('Error!') candidate = cands results = utils.eval_metrics(reference, candidate, label_dict, opt.log) results = [('%s: %.5f' % item + '\n') for item in results.items()] with codecs.open(opt.log + 'results.txt', 'w', 'utf-8') as f: f.writelines(results)
def evaluate(self): if self.val_loader is None: self.logger.warning('Not data loader was passed for the validation step, No validation is performed !') return {} self.logger.info('\n###### EVALUATION ######') self.model.eval() self.wrt_mode = 'val' self._eval_metrics() tbar = tqdm(self.val_loader, ncols=130) with torch.no_grad(): for batch_idx, sample in enumerate(tbar): torch.cuda.synchronize(device=self.device) tic = time.time() data, target = sample['img'].to(self.device), sample['label'].to(self.device) # LOSS if self.config['model']['name'] == 'UNet_ConvLSTM': output, _ = self.model(data) else: output = self.model(data.to(self.device)) assert output.size()[2:] == target.size()[1:] assert output.size()[1] == self.num_classes # update time torch.cuda.synchronize(self.device) self.batch_time.update(time.time() - tic) seg_metrics = eval_metrics(output, target, self.num_classes) self._update_seg_metrics(*seg_metrics) # PRINT INFO pixAcc, mIoU, _ = self._get_seg_metrics().values() tbar.set_description('PixelAcc: {:.2f}, Mean IoU: {:.2f}, Inference Time: {:.4f} |' .format(pixAcc, mIoU, self.batch_time.average)) # METRICS TO TENSORBOARD seg_metrics = self._get_seg_metrics() log = { 'batch_time': self.batch_time.average, **seg_metrics } return log
def eval_model(model, data, params, config, device, writer): model.eval() reference, candidate, source, alignments = [], [], [], [] # count, total_count = 0, len(data['valid_set']) valid_loader = data["valid_loader"] tgt_vocab = data["tgt_vocab"] for src, tgt, src_len, tgt_ylen, original_src, original_tgt, knowledge, knowledge_len in tqdm( valid_loader): # print(original_src) src = src.to(device) src_len = src_len.to(device) if config.knowledge: knowledge, knowledge_len = knowledge.to(device), knowledge_len.to( device) with torch.no_grad(): if config.beam_size > 1: samples, alignment = model.beam_sample( src, src_len, knowledge, knowledge_len, beam_size=config.beam_size, eval_=False) else: samples, alignment = model.sample(src, src_len, knowledge, knowledge_len) candidate += [tgt_vocab.convertToLabels(s, utils.EOS) for s in samples] source += original_src reference += original_tgt if alignment is not None: alignments += [align for align in alignment] if config.unk and config.attention != "None": cands = [] for s, c, align in zip(source, candidate, alignments): cand = [] for word, idx in zip(c, align): if word == utils.UNK_WORD and idx < len(s): try: cand.append(s[idx]) except: cand.append(word) print("%d %d\n" % (len(s), idx)) else: cand.append(word) cands.append(cand) if len(cand) == 0: print("Error!") candidate = cands with codecs.open(os.path.join(params["log_path"], "candidate.txt"), "w+", "utf-8") as f: for i in range(len(candidate)): f.write(f"{' '.join(candidate[i])}\n") post_process.del_repeat(os.path.join(params["log_path"], "candidate.txt"), os.path.join(params["log_path"], "output.txt")) if config.label_dict_file != "": results = utils.eval_metrics(reference, candidate, label_dict, params["log_path"]) score = {} result_line = "" for metric in config.metrics: if config.label_dict_file != "": score[metric] = results[metric] result_line += metric + ": %s " % str(score[metric]) else: score[metric] = getattr(utils, metric)(reference, candidate, params["log_path"], print, config) if config.label_dict_file != "": result_line += "\n" print(result_line) return score
def test(args, epoch, eval_alpha=0.5): print('Evaluating for epoch = %d' % epoch) losses, psnrs, ssims, lpips = utils.init_meters(args.loss) model.eval() criterion.eval() save_folder = 'test%03d' % epoch if args.dataset == 'snufilm': save_folder = os.path.join(save_folder, args.dataset, args.test_mode) else: save_folder = os.path.join(save_folder, args.dataset) save_dir = os.path.join('checkpoint', args.exp_name, save_folder) utils.makedirs(save_dir) save_fn = os.path.join(save_dir, 'results.txt') if not os.path.exists(save_fn): with open(save_fn, 'w') as f: f.write('For epoch=%d\n' % epoch) t = time.time() with torch.no_grad(): for i, (images, imgpaths) in enumerate(tqdm(test_loader)): # Build input batch im1, im2, gt = utils.build_input(images, imgpaths, is_training=False) # Forward out, feats = model(im1, im2) # Save loss values loss, loss_specific = criterion(out, gt, None, feats) for k, v in losses.items(): if k != 'total': v.update(loss_specific[k].item()) losses['total'].update(loss.item()) # Evaluate metrics utils.eval_metrics(out, gt, psnrs, ssims, lpips) # Log examples that have bad performance if (ssims.val < 0.9 or psnrs.val < 25) and epoch > 50: print(imgpaths) print("\nLoss: %f, PSNR: %f, SSIM: %f, LPIPS: %f" % (losses['total'].val, psnrs.val, ssims.val, lpips.val)) print(imgpaths[1][-1]) # Save result images if ((epoch + 1) % 1 == 0 and i < 20) or args.mode == 'test': savepath = os.path.join('checkpoint', args.exp_name, save_folder) for b in range(images[0].size(0)): paths = imgpaths[1][b].split('/') fp = os.path.join(savepath, paths[-3], paths[-2]) if not os.path.exists(fp): os.makedirs(fp) # remove '.png' extension fp = os.path.join(fp, paths[-1][:-4]) utils.save_image(out[b], "%s.png" % fp) # Print progress print('im_processed: {:d}/{:d} {:.3f}s \r'.format(i + 1, len(test_loader), time.time() - t)) print("Loss: %f, PSNR: %f, SSIM: %f, LPIPS: %f\n" % (losses['total'].avg, psnrs.avg, ssims.avg, lpips.avg)) # Save psnr & ssim save_fn = os.path.join('checkpoint', args.exp_name, save_folder, 'results.txt') with open(save_fn, 'a') as f: f.write("PSNR: %f, SSIM: %f, LPIPS: %f\n" % (psnrs.avg, ssims.avg, lpips.avg)) # Log to TensorBoard if args.mode != 'test': utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg, optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i, mode='test') return losses['total'].avg, psnrs.avg, ssims.avg, lpips.avg
def test(): df_column = ['Name'] df_column.extend([str(i) for i in range(1, seq_len + 1)]) df = pd.DataFrame(columns=df_column) psnr_array = np.zeros((0, seq_len)) ssim_array = np.zeros((0, seq_len)) tqdm_loader = tqdm.tqdm(validationloader, ncols=80) imgsave_folder = os.path.join(args.checkpoint_dir, 'Saved_imgs') if not os.path.exists(imgsave_folder): os.mkdir(imgsave_folder) with torch.no_grad(): for validationIndex, (validationData, validationFrameIndex, validationFile) in enumerate(tqdm_loader): blurred_img = torch.zeros_like(validationData[0]) for image in validationData: blurred_img += image blurred_img /= len(validationData) blurred_img = blurred_img.to(device) batch_size = blurred_img.shape[0] blurred_img = meanshift(blurred_img, mean, std, device, False) c = center_estimation(blurred_img) start, end = border_estimation(blurred_img, c) start = meanshift(start, mean, std, device, True) end = meanshift(end, mean, std, device, True) blurred_img = meanshift(blurred_img, mean, std, device, True) frame0 = validationData[0].to(device) frame1 = validationData[-1].to(device) batch_size = blurred_img.shape[0] parallel = torch.mean(compare_ftn(start, frame0) + compare_ftn(end, frame1), dim=(1, 2, 3)) cross = torch.mean(compare_ftn(start, frame1) + compare_ftn(end, frame0), dim=(1, 2, 3)) I0 = torch.zeros_like(blurred_img) I1 = torch.zeros_like(blurred_img) for b in range(batch_size): if parallel[b] <= cross[b]: I0[b], I1[b] = start[b], end[b] else: I0[b], I1[b] = end[b], start[b] psnrs = np.zeros((batch_size, seq_len)) ssims = np.zeros((batch_size, seq_len)) for vindex in range(seq_len): frameT = validationData[vindex] IFrame = frameT.to(device) if vindex == 0: Ft_p = I0.clone() elif vindex == seq_len - 1: Ft_p = I1.clone() else: validationIndex = torch.ones(batch_size) * (vindex - 1) validationIndex = validationIndex.long() flowOut = flowComp(torch.cat((I0, I1), dim=1)) F_0_1 = flowOut[:, :2, :, :] F_1_0 = flowOut[:, 2:, :, :] fCoeff = superslomo.getFlowCoeff(validationIndex, device, seq_len) F_t_0 = fCoeff[0] * F_0_1 + fCoeff[1] * F_1_0 F_t_1 = fCoeff[2] * F_0_1 + fCoeff[3] * F_1_0 g_I0_F_t_0 = validationFlowBackWarp(I0, F_t_0) g_I1_F_t_1 = validationFlowBackWarp(I1, F_t_1) if args.add_blur: intrpOut = ArbTimeFlowIntrp( torch.cat((I0, I1, F_0_1, F_1_0, F_t_1, F_t_0, g_I1_F_t_1, g_I0_F_t_0, blurred_img), dim=1)) else: intrpOut = ArbTimeFlowIntrp( torch.cat((I0, I1, F_0_1, F_1_0, F_t_1, F_t_0, g_I1_F_t_1, g_I0_F_t_0), dim=1)) F_t_0_f = intrpOut[:, :2, :, :] + F_t_0 F_t_1_f = intrpOut[:, 2:4, :, :] + F_t_1 V_t_0 = torch.sigmoid(intrpOut[:, 4:5, :, :]) V_t_1 = 1 - V_t_0 g_I0_F_t_0_f = validationFlowBackWarp(I0, F_t_0_f) g_I1_F_t_1_f = validationFlowBackWarp(I1, F_t_1_f) wCoeff = superslomo.getWarpCoeff(validationIndex, device, seq_len) Ft_p = (wCoeff[0] * V_t_0 * g_I0_F_t_0_f + wCoeff[1] * V_t_1 * g_I1_F_t_1_f) / ( wCoeff[0] * V_t_0 + wCoeff[1] * V_t_1) Ft_p = meanshift(Ft_p, mean, std, device, False) IFrame = meanshift(IFrame, mean, std, device, False) for b in range(batch_size): foldername = os.path.basename( os.path.dirname(validationFile[ctr_idx][b])) filename = os.path.splitext( os.path.basename(validationFile[vindex][b]))[0] out_fname = foldername + '_' + filename + '_out.png' gt_fname = foldername + '_' + filename + '.png' out, gt = quantize(Ft_p[b]), quantize(IFrame[b]) # Comment two lines below if you want to save images # torchvision.utils.save_image(out, os.path.join(imgsave_folder, out_fname), normalize=True, range=(0,255)) # torchvision.utils.save_image(gt, os.path.join(imgsave_folder, gt_fname), normalize=True, range=(0,255)) psnr, ssim = eval_metrics(Ft_p, IFrame) psnrs[:, vindex] = psnr.cpu().numpy() ssims[:, vindex] = ssim.cpu().numpy() for b in range(batch_size): rows = [validationFile[ctr_idx][b]] rows.extend(list(psnrs[b])) df = df.append(pd.Series(rows, index=df.columns), ignore_index=True) df.to_csv('{}/results_PSNR.csv'.format(args.checkpoint_dir))
def eval_model(model, data, params): model.eval() reference, candidate, source, alignments = [], [], [], [] count, total_count = 0, len(data['validset']) validloader = data['validloader'] tgt_vocab = data['tgt_vocab'] for src, tgt, src_len, tgt_len, original_src, original_tgt in validloader: if config.use_cuda: src = src.cuda() src_len = src_len.cuda() with torch.no_grad(): if config.beam_size > 1: samples, alignment, weight = model.beam_sample( src, src_len, beam_size=config.beam_size, eval_=True) else: samples, alignment = model.sample(src, src_len) candidate += [tgt_vocab.convertToLabels(s, utils.EOS) for s in samples] source += original_src reference += original_tgt if alignment is not None: alignments += [align for align in alignment] count += len(original_src) utils.progress_bar(count, total_count) if config.unk and config.attention != 'None': cands = [] for s, c, align in zip(source, candidate, alignments): cand = [] for word, idx in zip(c, align): if word == utils.UNK_WORD and idx < len(s): try: cand.append(s[idx]) except: cand.append(word) print("%d %d\n" % (len(s), idx)) else: cand.append(word) cands.append(cand) if len(cand) == 0: print('Error!') candidate = cands with codecs.open(params['log_path'] + 'candidate.txt', 'w+', 'utf-8') as f: for i in range(len(candidate)): f.write(" ".join(candidate[i]) + '\n') results = utils.eval_metrics(reference, candidate, label_dict, params['log_path']) score = {} result_line = "" for metric in config.metrics: score[metric] = results[metric] result_line += metric + ": %s " % str(score[metric]) result_line += '\n' params['log'](result_line) return score
def _train_epoch(self, epoch): self.logger.info('\n') self.model.train() if self.config['model']['args']['freeze_bn']: if isinstance(self.model, torch.nn.DataParallel): self.model.module.freeze_bn() else: self.model.freeze_bn() self.wrt_mode = 'train' tic = time.time() tbar = tqdm(self.train_loader, ncols=130) self._reset_metrics() for batch_idx, sample in enumerate(tbar): torch.cuda.synchronize(device=self.device) self.data_time.update(time.time() - tic) self.optimizer.zero_grad() if self.prefetch: data = sample['img'] target = sample['label'] else: data = sample['img'].to(self.device) target = sample['label'].to(self.device) if self.config['model']['name'] == 'UNet_ConvLSTM': output, _ = self.model(data) else: output = self.model(data) # print(output.size(), target.size()) assert output.size()[2:] == target.size()[1:] assert output.size()[1] == self.num_classes loss = self.loss(output, target) if isinstance(self.loss, torch.nn.DataParallel): loss = loss.mean() loss.backward() self.optimizer.step() self.total_loss.update(loss.item()) self.lr_scheduler.step(epoch=epoch - 2) # measure time torch.cuda.synchronize(device=self.device) self.batch_time.update(time.time() - tic) tic = time.time() if batch_idx % self.log_step == 0: self.wrt_step = (epoch - 1) * len( self.train_loader) + batch_idx self.writer.add_scalar('{}/loss'.format(self.wrt_mode), loss.item(), self.wrt_step) # FOR EVAL seg_metrics = eval_metrics(output, target, self.num_classes, is_train=True) self._update_seg_metrics(*seg_metrics) pixAcc, mIoU, _ = self._get_seg_metrics().values() # PRINT INFO tbar.set_description( 'TRAIN ({}) | Loss: {} | Acc: {:.4f} mIoU: {:.4f}'.format( epoch, self.total_loss.average, pixAcc, mIoU)) # METRICS TO TENSORBOARD seg_metrics = self._get_seg_metrics() for k, v in list(seg_metrics.items())[:-1]: self.writer.add_scalar('{}/{}'.format(self.wrt_mode, k), v, self.wrt_step) for i, opt_group in enumerate(self.optimizer.param_groups): self.writer.add_scalar( '{}/Learning_rate_{}'.format(self.wrt_mode, i), opt_group['lr'], self.wrt_step) # self.writer.add_scalar(f'{self.wrt_mode}/Momentum_{k}', opt_group['momentum'], self.wrt_step) # RETURN LOSS & METRICS log = {'loss': self.total_loss.average, **seg_metrics} # if self.lr_scheduler is not None: self.lr_scheduler.step() return log
def _valid_epoch(self, epoch): if self.val_loader is None: self.logger.warning( 'Not data loader was passed for the validation step, No validation is performed !' ) return {} self.logger.info('\n###### EVALUATION ######') self.model.eval() self.wrt_mode = 'val' self._reset_metrics() tbar = tqdm(self.val_loader, ncols=130) with torch.no_grad(): val_visual = [] for batch_idx, sample in enumerate(tbar): if self.prefetch: data = sample['img'] target = sample['label'] else: data = sample['img'].to(self.device) target = sample['label'].to(self.device) if self.config['model']['name'] == 'UNet_ConvLSTM': output, _ = self.model(data) else: output = self.model(data) assert output.size()[2:] == target.size()[1:] assert output.size()[1] == self.num_classes loss = self.loss(output, target) if isinstance(self.loss, torch.nn.DataParallel): loss = loss.mean() self.total_loss.update(loss.item()) seg_metrics = eval_metrics(output, target, self.num_classes, is_train=True) # seg_metrics = seg_metrics[:4] self._update_seg_metrics(*seg_metrics) # LIST OF IMAGE TO VIZ (15 images) if len(val_visual) < 15: target_np = target.data.cpu().numpy() output_np = output.data.max(1)[1].cpu().numpy() if self.config['dataset']['name'] == "SeqLane": val_visual.append([ sample['img'][0][-1].data.cpu(), target_np[0], output_np[0] ]) else: val_visual.append([ sample['img'][0].data.cpu(), target_np[0], output_np[0] ]) # PRINT INFO pixAcc, mIoU, _ = self._get_seg_metrics().values() tbar.set_description( 'EVAL ({}) | Loss: {:.3f}, PixelAcc: {:.2f}, Mean IoU: {:.2f} |' .format(epoch, self.total_loss.average, pixAcc, mIoU)) # WRTING & VISUALIZING THE MASKS val_img = get_val_image(val_visual, self.MEAN, self.STD, self.val_loader.dataset.palette) self.writer.add_image( '{}/inputs_targets_predictions'.format(self.wrt_mode), val_img, self.wrt_step) # METRICS TO TENSORBOARD self.wrt_step = (epoch) * len(self.val_loader) self.writer.add_scalar('{}/loss'.format(self.wrt_mode), self.total_loss.average, self.wrt_step) seg_metrics = self._get_seg_metrics() for k, v in list(seg_metrics.items())[:-1]: self.writer.add_scalar('{}/{}'.format(self.wrt_mode, k), v, self.wrt_step) log = {'val_loss': self.total_loss.average, **seg_metrics} return log
for batch_idx, batch in enumerate(_trange): batch = [b.to("cuda") for b in batch] data, target = batch[:-1], batch[-1] # clear the gradients of all optimized variables optimizer.zero_grad() # forward pass: compute predicted outputs by passing inputs to the model output = model(data) # calculate the loss loss = loss_fn.forward(output, target) loss.backward() # perform a single optimization step (parameter update) optimizer.step() # update running training loss train_loss += loss.item() total_metrics += eval_metrics(output, target, metrics, writer) if batch_idx % int(np.sqrt(dataloader.batch_size)) == 0: _str = 'Train Epoch: {} Loss: {:.6f}'.format(epoch,loss.item()) _trange.set_description(_str) # print training statistics # calculate average loss over an epoch # Add epoch metrics loss = train_loss / len(dataloader) metric_epoch = (total_metrics / len(dataloader)).tolist() writer.add_scalar('loss', loss) for i, metric in enumerate(metrics): writer.add_scalar("%s"%metric.__name__, metric_epoch[i])
def eval_detector(args): load_model_dir = args.load_model_dir load_model_name = args.load_model_name normal_data_dir = args.normal_data_dir normal_data_name_train = args.normal_data_name_train normal_data_name_test = args.normal_data_name_test abnormal_data_dir = args.abnormal_data_dir abnormal_data_name = args.abnormal_data_name Pvalue_th = args.Pvalue_th val_and_ref_name = args.normal_data_name_val_and_ref gpu = args.gpu AnomalyDetector = torch.load(load_model_dir + load_model_name) AnomalyDetector.eval() AnomalyDetector.th = Pvalue_th if args.dummydata: _, _, testing_normal_data = loaddata.load_normal_dummydata() else: if args.debug: _, training_normal_data, _, = (loaddata.load_data_split( data_dir=normal_data_dir, file_name=normal_data_name_train, split=(0.1, 0.8, 0.1))) _, ref_normal_data, val_normal_data = loaddata.load_data_split( data_dir=normal_data_dir, file_name=val_and_ref_name, split=(0.1, 0.45, 0.45)) training_normal_data = torch.tensor( AnomalyDetector.normalize(training_normal_data)) val_normal_data = torch.tensor( AnomalyDetector.normalize(val_normal_data)) ref_normal_data = torch.tensor( AnomalyDetector.normalize(ref_normal_data)) testing_normal_data = loaddata.load_data_all( data_dir=normal_data_dir, file_name=normal_data_name_test) testing_normal_data = torch.tensor( AnomalyDetector.normalize(testing_normal_data)) if args.dummydata: testing_abnormal_data = loaddata.load_abnormal_dummydata() else: testing_abnormal_data = loaddata.load_data_all( data_dir=abnormal_data_dir, file_name=abnormal_data_name) testing_abnormal_data = torch.tensor( AnomalyDetector.normalize(testing_abnormal_data)) print("testing_abnormal_data.shape ", testing_abnormal_data.shape) if gpu: AnomalyDetector = AnomalyDetector.cuda() testing_normal_data = testing_normal_data.cuda() testing_abnormal_data = testing_abnormal_data.cuda() true_label_normal = np.zeros( len(testing_normal_data) - AnomalyDetector.RED_collection_len * AnomalyDetector.RED_points - 1) true_label_abnormal = np.ones( len(testing_abnormal_data) - AnomalyDetector.RED_collection_len * AnomalyDetector.RED_points - 1) true_label = np.concatenate((true_label_normal, true_label_abnormal), axis=0) pred_normal, p_values_normal = AnomalyDetector.predict(testing_normal_data, gpu, debug=args.debug) if args.debug: feature_idx = 0 # debug_pred_normal is of size [seq_len-1, batch(=1), features] RE_normal, debug_pred_normal = AnomalyDetector._get_reconstruction_error( testing_normal_data, gpu=gpu) seq_dict = { "truth": testing_normal_data[1:, feature_idx].detach().numpy(), "pred": debug_pred_normal[:, 0, feature_idx].detach().numpy(), } seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2 utils.plot_seq(seq_dict, title="Testing normal prediction") # debug_pred_normal is of size [seq_len-1, batch(=1), features] RE_abnormal, debug_pred_abnormal = AnomalyDetector._get_reconstruction_error( testing_abnormal_data, gpu=gpu) seq_dict = { "truth": testing_abnormal_data[1:, feature_idx].detach().numpy(), "pred": debug_pred_abnormal[:, 0, feature_idx].detach().numpy(), } seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2 utils.plot_seq(seq_dict, title="Testing abnormal prediction") # debug_ref is of size [seq_len-1, batch(=1), features] RE_ref, debug_ref = AnomalyDetector._get_reconstruction_error( ref_normal_data, gpu=gpu) seq_dict = { "truth": ref_normal_data[1:, feature_idx].detach().numpy(), "pred": debug_ref[:, 0, feature_idx].detach().numpy(), } seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2 utils.plot_seq(seq_dict, title="Train normal ref prediction") RE_seq_dict = { "RE_reference": RE_ref, "RE_normal": RE_normal, "RE_abnormal": RE_abnormal } utils.plot_seq(RE_seq_dict, title="Reconstruction errors") utils.plot_cdf(RE_seq_dict, title="RED cdf") print("p_values_normal.shape ", len(p_values_normal)) print("p_values_normal.mean ", np.mean(p_values_normal)) pred_abnormal, p_values_abnormal = AnomalyDetector.predict( testing_abnormal_data, gpu, debug=args.debug) print("p_values_abnormal.shape ", len(p_values_abnormal)) print("p_values_abnormal.mean ", np.mean(p_values_abnormal)) pred = np.concatenate((pred_normal, pred_abnormal), axis=0) pred_score = np.concatenate((p_values_normal, p_values_abnormal), axis=0) print("true_label.shape", true_label.shape, "pred.shape", pred.shape) tp, fp, fn, tn, acc, prec, rec, f1, fpr, tpr, thresholds, roc_auc = ( utils.eval_metrics( truth=true_label, pred=pred, anomaly_score=-np.log10( pred_score + 1e-500) # Anomaly score=-log(p_value) )) plt.figure() plt.plot(fpr, tpr, color='darkorange') plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--') plt.xlim([-0.01, 1.01]) plt.ylim([-0.01, 1.01]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC of LSTM anomaly detector') plt.legend(loc="lower right") plt.show() return roc_auc
def run_train(det_save_path, n_classes, from_tfr): graph = tf.get_default_graph() sess = tf.Session(graph=graph) if from_tfr: C_dataset = TFR_Dataset(FLAGS.epochs, FLAGS.batch_size, FLAGS.prefetch_num, TRAIN_ROOT, 10000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, file_shards=FLAGS.train_shards, augment=True, preprocess=True) C_dataset.make_tfr_dataset() else: C_dataset = Memory_Dataset(FLAGS.epochs, FLAGS.batch_size, FLAGS.prefetch_num, UP_TRAIN_PRECROP, 2000, PRECROP_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, preprocess=True) C_dataset.make_memory_dataset(aug_crop=True) fluid_iterator = Dataset_iterator(C_dataset.dataset) el = fluid_iterator.get_el() net = Recognition_Network(n_classes, FLAGS.box_gamma, FLAGS.alpha_tail, sess, graph, P_VALUE, alpha_head=FLAGS.alpha_head) net.set_params(el) net.build_optimizer(lr=FLAGS.learning_rate, resnet_path=RESNET_MODEL, patch_model_path=PATCH_MODEL, total_model_path=TOTAL_MODEL) step = 0 if from_tfr: net.sess.run( fluid_iterator.return_initialized_iterator(C_dataset.dataset)) else: C_dataset.make_memory_feed_dict() net.sess.run(fluid_iterator.return_initialized_iterator( C_dataset.dataset), feed_dict=C_dataset.feed_dict) train_start_time = time.time() interval_time = train_start_time try: while True: _, losses = net.sess.run([net.train_op, net.losses]) if step % FLAGS.print_step == 0: # ~46000 cases / batch size of 6 ~= 7.7e3 steps per epoch # ~ 1800 / 6 ~= 300 steps per epoch for UP cases c_end = time.time() from_start = (c_end - train_start_time ) / 3600 # minutes and seconds to hours from_last_step = (c_end - interval_time) / 60 # seconds to hours print( 'Step {0:g}, error {1:.3f}, {2:.2f} hrs from start, {3:.2f} mins from last epoch' .format(step, losses['total'], from_start, from_last_step)) interval_time = c_end step += 1 if step % FLAGS.save_step == 0: # 7700 steps per epochs * 40 epochs = 3.0e5 steps # 300 steps/epoch * 2000 epochs for UP cases = 4.5e5 steps, just save only at end net.total_model_saver.save(net.sess, det_save_path, global_step=step) except tf.errors.OutOfRangeError: net.total_model_saver.save(net.sess, det_save_path, global_step=step) final_timing = (time.time() - train_start_time) / 3600 print('Done training after total time of {0:.1f} hours'.format( final_timing)) print('L2 loss tensors are: {}'.format( graph.get_collection('l2_loss'))) # Scoring; no longer in training with BN if from_tfr: # need to reset C_dataset to change epoch C_dataset = TFR_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, TRAIN_ROOT, 10000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, file_shards=FLAGS.train_shards, augment=False, preprocess=False) C_dataset.make_tfr_dataset() net.sess.run( fluid_iterator.return_initialized_iterator(C_dataset.dataset)) else: C_dataset = Memory_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, UP_TRAIN_300, 2000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, preprocess=False) C_dataset.make_memory_dataset() C_dataset.make_memory_feed_dict() net.sess.run(fluid_iterator.return_initialized_iterator( C_dataset.dataset), feed_dict=C_dataset.feed_dict) pred_dict, labels_list = create_holders(n_classes) while True: try: scores, singles = net.score() for i, j in scores.items(): pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0) labels_list = np.concatenate([labels_list, singles], axis=0) except tf.errors.OutOfRangeError: break print('The training metrics are: ') eval_metrics(pred_dict, labels_list) #Scoring for test set; if from_tfr: C_dataset = TFR_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, TEST_ROOT, 10000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, file_shards=FLAGS.train_shards, augment=False, preprocess=False) C_dataset.make_tfr_dataset() net.sess.run( fluid_iterator.return_initialized_iterator(C_dataset.dataset)) else: C_dataset = Memory_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num, UP_TEST, 2000, TARGET_IMG_SIZE, TARGET_IMG_SIZE, LABEL_SIZE, SINGLES, preprocess=False) C_dataset.make_memory_dataset() C_dataset.make_memory_feed_dict() net.sess.run(fluid_iterator.return_initialized_iterator( C_dataset.dataset), feed_dict=C_dataset.feed_dict) pred_dict, labels_list = create_holders(n_classes) while True: try: scores, singles = net.score() for i, j in scores.items(): pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0) labels_list = np.concatenate([labels_list, singles], axis=0) except tf.errors.OutOfRangeError: break print('The Testing metrics are: ') eval_metrics(pred_dict, labels_list)
def train(args, epoch): ### GUI things global psnrs global out global gt global it global apptr ### progres bar global startedpbar startedpbar=0 ### loss global LOSS_0 losses, psnrs, ssims, lpips = utils.init_meters(args.loss) model.train() criterion.train() t = time.time() for i, (images, imgpaths) in enumerate(train_loader): #print(startedpbar) if startedpbar==0: pbar=tqdm(range(i, len(train_loader))) startedpbar=1 else: startedpbar=1 #print(startedpbar) # Build input batch im1, im2, gt = utils.build_input(images, imgpaths) # Forward optimizer.zero_grad() out, feats = model(im1, im2) it+=1 loss, loss_specific = criterion(out, gt, None, feats) QApplication.processEvents() # Save loss values for k, v in losses.items(): if k != 'total': v.update(loss_specific[k].item()) if LOSS_0 == 0: LOSS_0 = loss.data.item() losses['total'].update(loss.item()) # Backward (+ grad clip) - if loss explodes, skip current iteration loss.backward() if loss.data.item() > 10.0 * LOSS_0: print(max(p.grad.data.abs().max() for p in model.parameters())) continue torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1) optimizer.step() # Calc metrics & print logs if i % args.log_iter == 0: utils.eval_metrics(out, gt, psnrs, ssims, lpips, lpips_model) pbar.update(1) pbar.set_postfix({'psnr': psnrs.avg, 'Loss': losses['total'].avg, 'epoch': epoch, 'iterations': it }) # Log to TensorBoard utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg, optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i) # Reset metrics t = time.time() # update gui if args.gui=="True": tgui = threading.Thread(target=updategui) tgui.start() losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
def validate(model, epoch): val_dataset = VimeoSeptuplet( cfg ) #('data/vimeo_septuplet', [1, 2, 3, 4, 5, 6, 7], is_training=False) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False, num_workers=8, pin_memory=True) #model.eval() if True: #with torch.no_grad(): losses = utils.AverageMeter() psnrs = utils.AverageMeter() ssims = utils.AverageMeter() batch_time = utils.AverageMeter() t = time.time() for i, (images, metadata) in enumerate(tqdm(val_loader)): if i == VAL_ITER_CUT: break t = time.time() images = [im.cuda() for im in images] target = images[3] weights_before = copy.deepcopy(net.state_dict()) val_optimizer = torch.optim.Adamax(net.parameters(), lr=cfg.inner_lr) # inner loop k = cfg.num_inner_update for _k in range(k): indices = [[0, 2, 4], [2, 4, 6]] total_loss = 0 for ind in indices: output = net(images[ind[0]].clone(), images[ind[2]].clone()) loss = criterion(output, images[ind[1]]) total_loss = total_loss + loss val_optimizer.zero_grad() total_loss.backward() val_optimizer.step() with torch.no_grad(): input0, input1, target = images[2], images[4], images[3] output = model(input0, input1) batch_time.update(time.time() - t) loss = criterion(output, target) losses.update(loss.item(), input0.size(0)) utils.eval_metrics(output, target, psnrs, ssims, None) print(psnrs.val, ssims.val) if VIZ: for b in range(images[0].size(0)): imgpath = metadata['imgpaths'][0][b] #print(imgpath) savepath = os.path.join('checkpoint', cfg.exp_name, 'vimeoSeptuplet', imgpath.split('/')[-3], imgpath.split('/')[-2]) if not os.path.exists(savepath): #print('make dirs... %s' % savepath) os.makedirs(savepath) img_pred = (output[b].data.permute(1, 2, 0).clamp_( 0, 1).cpu().numpy()[..., ::-1] * 255).astype( numpy.uint8) cv2.imwrite(os.path.join(savepath, 'im2_pred.png'), img_pred) #batch_time.update(time.time() - t) #t = time.time() # restore the original base weight net.load_state_dict(weights_before) print("val_losses: %f" % losses.avg) print("val_PSNR: %f, val_SSIM: %f" % (psnrs.avg, ssims.avg)) print("Time per batch: %.3f" % batch_time.avg) return psnrs.avg
def train(epoch): print("Training Epoch: %d" % epoch) net.train() losses = utils.AverageMeter() psnrs = utils.AverageMeter() ssims = utils.AverageMeter() batch_time = utils.AverageMeter() t = time.time() for i, (images, _) in enumerate(train_loader): if i == 4000: break images = [im.cuda() for im in images] # Meta training k = cfg.num_inner_update # inner update iteration if cfg.meta_algorithm == 'reptile': weights_before = copy.deepcopy(net.state_dict()) inner_optimizer = torch.optim.Adamax(net.parameters(), lr=cfg.inner_lr) # inner loop for _k in range(k): indices = [[0, 2, 4], [2, 4, 6]] total_loss = 0 for ind in indices: output = net(images[ind[0]].clone(), images[ind[2]].clone()) loss = criterion(output, images[ind[1]]) total_loss = total_loss + loss inner_optimizer.zero_grad() total_loss.backward() inner_optimizer.step() # Reptile - outer update outerstepsize = cfg.outer_lr weights_after = net.state_dict() net.load_state_dict({ name: weights_before[name] + (weights_after[name] - weights_before[name]) * outerstepsize for name in weights_before }) # calculate loss w/ updated model input0, input1, target = images[2], images[4], images[3] with torch.no_grad(): output = net(input0, input1) loss = criterion(output, target) elif cfg.meta_algorithm == 'maml': base_net = copy.deepcopy(net) inner_optimizer = torch.optim.Adamax(net.parameters(), lr=cfg.inner_lr) # inner loop for _k in range(k): indices = [[0, 2, 4], [2, 4, 6]] total_loss = 0 for ind in indices: output = net(images[ind[0]].clone(), images[ind[2]].clone()) loss = criterion(output, images[ind[1]]) total_loss = total_loss + loss inner_optimizer.zero_grad() total_loss.backward() inner_optimizer.step() # Forward on query data outerstepsize = cfg.outer_lr input0, input1, target = images[2], images[4], images[3] output = net(input0, input1) loss = criterion(output, target) # Copy base parameters to 'net' to connect the computation graph for param, base_param in zip(net.parameters(), base_net.parameters()): param.data = base_param.data # Calculate gradient & update meta-learner optimizer.zero_grad() grads = torch.autograd.grad(loss, net.parameters()) for j, param in enumerate(net.parameters()): #param = param - grads[j] * outerstepsize param.grad = grads[j] optimizer.step() losses.update(loss.item(), images[0].size(0)) batch_time.update(time.time() - t) t = time.time() # Logging if i % cfg.logfreq == 0: utils.eval_metrics(output, target, psnrs, ssims, None) print(( 'Epoch: [%d][%d/%d],\tTime %.3f (%.3f)\tLoss %.4f (%.4f)\tPSNR %.2f\t' % (epoch, i, len(train_loader), batch_time.val, batch_time.avg, losses.val, losses.avg, psnrs.val))) losses.reset() psnrs.reset() ssims.reset() batch_time.reset()