def run_infer(n_classes, from_tfr, in_memory_data=None, return_vals=False):
graph = tf.get_default_graph()
sess = tf.Session(graph=graph)
if from_tfr:
C_dataset = TFR_Dataset(1,
FLAGS.infer_batch,
FLAGS.prefetch_num,
TEST_ROOT,
10000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
file_shards=FLAGS.train_shards)
C_dataset.make_tfr_dataset()
else:
C_dataset = Memory_Dataset(1, FLAGS.infer_batch, FLAGS.prefetch_num,
UP_TEST, 2000, TARGET_IMG_SIZE,
TARGET_IMG_SIZE, LABEL_SIZE, SINGLES)
C_dataset.make_memory_dataset()
C_dataset.make_memory_feed_dict(in_memory_data)
fluid_iterator = Dataset_iterator(C_dataset.dataset)
el = fluid_iterator.get_el()
net = Recognition_Network(n_classes, 3, 0.05, sess, graph, P_VALUE)
net.set_params(el)
net.build_optimizer(lr=0.01,
resnet_path=None,
patch_model_path=PATCH_MODEL,
total_model_path=None)
if from_tfr:
sess.run(fluid_iterator.return_initialized_iterator(C_dataset.dataset))
else:
sess.run(fluid_iterator.return_initialized_iterator(C_dataset.dataset),
feed_dict=C_dataset.feed_dict)
pred_dict, labels_list = create_holders(n_classes)
while True:
try:
scores, singles = net.score()
for i, j in scores.items():
pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0)
labels_list = np.concatenate([labels_list, singles], axis=0)
except tf.errors.OutOfRangeError:
break
print('The metrics are: ')
eval_metrics(pred_dict, labels_list, plot_auc=False)
if return_vals:
return pred_dict, labels_list
def train(args, epoch):
global LOSS_0
losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
model.train()
criterion.train()
t = time.time()
for i, (images, imgpaths) in enumerate(train_loader):
# Build input batch
im1, im2, gt = utils.build_input(images, imgpaths)
# Forward
optimizer.zero_grad()
out, feats = model(im1, im2)
loss, loss_specific = criterion(out, gt, None, feats)
# Save loss values
for k, v in losses.items():
if k != 'total':
v.update(loss_specific[k].item())
if LOSS_0 == 0:
LOSS_0 = loss.data.item()
losses['total'].update(loss.item())
# Backward (+ grad clip) - if loss explodes, skip current iteration
loss.backward()
if loss.data.item() > 10.0 * LOSS_0:
print(max(p.grad.data.abs().max() for p in model.parameters()))
continue
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
optimizer.step()
# Calc metrics & print logs
if i % args.log_iter == 0:
utils.eval_metrics(out, gt, psnrs, ssims, lpips, lpips_model)
print('Train Epoch: {} [{}/{}]\tLoss: {:.6f}\tPSNR: {:.4f}\tTime({:.2f})'.format(
epoch, i, len(train_loader), losses['total'].avg, psnrs.avg, time.time() - t))
# Log to TensorBoard
utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg,
optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i)
# Reset metrics
losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
t = time.time()
def eval_model(model, data):
model.eval()
reference, candidate, source, alignments = [], [], [], []
tgt_vocab = data['tgt_vocab']
count, total_count = 0, len(data['testset'])
dataloader = data['testloader']
for src, tgt, src_len, tgt_len, original_src, original_tgt in dataloader:
if config.use_cuda:
src = src.cuda()
src_len = src_len.cuda()
with torch.no_grad():
if config.beam_size > 1 and (not config.global_emb):
samples, alignment, _ = model.beam_sample(
src, src_len, beam_size=config.beam_size, eval_=True)
else:
samples, alignment = model.sample(src, src_len)
candidate += [
tgt_vocab.convertToLabels(s.tolist(), utils.EOS) for s in samples
]
source += original_src
reference += original_tgt
if alignment is not None:
alignments += [align for align in alignment]
count += len(original_src)
utils.progress_bar(count, total_count)
if config.unk and config.attention != 'None':
cands = []
for s, c, align in zip(source, candidate, alignments):
cand = []
for word, idx in zip(c, align):
if word == utils.UNK_WORD and idx < len(s):
try:
cand.append(s[idx])
except:
cand.append(word)
print("%d %d\n" % (len(s), idx))
else:
cand.append(word)
cands.append(cand)
if len(cand) == 0:
print('Error!')
candidate = cands
results = utils.eval_metrics(reference, candidate, label_dict, opt.log)
results = [('%s: %.5f' % item + '\n') for item in results.items()]
with codecs.open(opt.log + 'results.txt', 'w', 'utf-8') as f:
f.writelines(results)
def evaluate(self):
if self.val_loader is None:
self.logger.warning('Not data loader was passed for the validation step, No validation is performed !')
return {}
self.logger.info('\n###### EVALUATION ######')
self.model.eval()
self.wrt_mode = 'val'
self._eval_metrics()
tbar = tqdm(self.val_loader, ncols=130)
with torch.no_grad():
for batch_idx, sample in enumerate(tbar):
torch.cuda.synchronize(device=self.device)
tic = time.time()
data, target = sample['img'].to(self.device), sample['label'].to(self.device)
# LOSS
if self.config['model']['name'] == 'UNet_ConvLSTM':
output, _ = self.model(data)
else:
output = self.model(data.to(self.device))
assert output.size()[2:] == target.size()[1:]
assert output.size()[1] == self.num_classes
# update time
torch.cuda.synchronize(self.device)
self.batch_time.update(time.time() - tic)
seg_metrics = eval_metrics(output, target, self.num_classes)
self._update_seg_metrics(*seg_metrics)
# PRINT INFO
pixAcc, mIoU, _ = self._get_seg_metrics().values()
tbar.set_description('PixelAcc: {:.2f}, Mean IoU: {:.2f}, Inference Time: {:.4f} |'
.format(pixAcc, mIoU, self.batch_time.average))
# METRICS TO TENSORBOARD
seg_metrics = self._get_seg_metrics()
log = {
'batch_time': self.batch_time.average,
**seg_metrics
}
return log
def eval_model(model, data, params, config, device, writer):
model.eval()
reference, candidate, source, alignments = [], [], [], []
# count, total_count = 0, len(data['valid_set'])
valid_loader = data["valid_loader"]
tgt_vocab = data["tgt_vocab"]
for src, tgt, src_len, tgt_ylen, original_src, original_tgt, knowledge, knowledge_len in tqdm(
valid_loader):
# print(original_src)
src = src.to(device)
src_len = src_len.to(device)
if config.knowledge:
knowledge, knowledge_len = knowledge.to(device), knowledge_len.to(
device)
with torch.no_grad():
if config.beam_size > 1:
samples, alignment = model.beam_sample(
src,
src_len,
knowledge,
knowledge_len,
beam_size=config.beam_size,
eval_=False)
else:
samples, alignment = model.sample(src, src_len, knowledge,
knowledge_len)
candidate += [tgt_vocab.convertToLabels(s, utils.EOS) for s in samples]
source += original_src
reference += original_tgt
if alignment is not None:
alignments += [align for align in alignment]
if config.unk and config.attention != "None":
cands = []
for s, c, align in zip(source, candidate, alignments):
cand = []
for word, idx in zip(c, align):
if word == utils.UNK_WORD and idx < len(s):
try:
cand.append(s[idx])
except:
cand.append(word)
print("%d %d\n" % (len(s), idx))
else:
cand.append(word)
cands.append(cand)
if len(cand) == 0:
print("Error!")
candidate = cands
with codecs.open(os.path.join(params["log_path"], "candidate.txt"), "w+",
"utf-8") as f:
for i in range(len(candidate)):
f.write(f"{' '.join(candidate[i])}\n")
post_process.del_repeat(os.path.join(params["log_path"], "candidate.txt"),
os.path.join(params["log_path"], "output.txt"))
if config.label_dict_file != "":
results = utils.eval_metrics(reference, candidate, label_dict,
params["log_path"])
score = {}
result_line = ""
for metric in config.metrics:
if config.label_dict_file != "":
score[metric] = results[metric]
result_line += metric + ": %s " % str(score[metric])
else:
score[metric] = getattr(utils,
metric)(reference, candidate,
params["log_path"], print, config)
if config.label_dict_file != "":
result_line += "\n"
print(result_line)
return score
def test(args, epoch, eval_alpha=0.5):
print('Evaluating for epoch = %d' % epoch)
losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
model.eval()
criterion.eval()
save_folder = 'test%03d' % epoch
if args.dataset == 'snufilm':
save_folder = os.path.join(save_folder, args.dataset, args.test_mode)
else:
save_folder = os.path.join(save_folder, args.dataset)
save_dir = os.path.join('checkpoint', args.exp_name, save_folder)
utils.makedirs(save_dir)
save_fn = os.path.join(save_dir, 'results.txt')
if not os.path.exists(save_fn):
with open(save_fn, 'w') as f:
f.write('For epoch=%d\n' % epoch)
t = time.time()
with torch.no_grad():
for i, (images, imgpaths) in enumerate(tqdm(test_loader)):
# Build input batch
im1, im2, gt = utils.build_input(images, imgpaths, is_training=False)
# Forward
out, feats = model(im1, im2)
# Save loss values
loss, loss_specific = criterion(out, gt, None, feats)
for k, v in losses.items():
if k != 'total':
v.update(loss_specific[k].item())
losses['total'].update(loss.item())
# Evaluate metrics
utils.eval_metrics(out, gt, psnrs, ssims, lpips)
# Log examples that have bad performance
if (ssims.val < 0.9 or psnrs.val < 25) and epoch > 50:
print(imgpaths)
print("\nLoss: %f, PSNR: %f, SSIM: %f, LPIPS: %f" %
(losses['total'].val, psnrs.val, ssims.val, lpips.val))
print(imgpaths[1][-1])
# Save result images
if ((epoch + 1) % 1 == 0 and i < 20) or args.mode == 'test':
savepath = os.path.join('checkpoint', args.exp_name, save_folder)
for b in range(images[0].size(0)):
paths = imgpaths[1][b].split('/')
fp = os.path.join(savepath, paths[-3], paths[-2])
if not os.path.exists(fp):
os.makedirs(fp)
# remove '.png' extension
fp = os.path.join(fp, paths[-1][:-4])
utils.save_image(out[b], "%s.png" % fp)
# Print progress
print('im_processed: {:d}/{:d} {:.3f}s \r'.format(i + 1, len(test_loader), time.time() - t))
print("Loss: %f, PSNR: %f, SSIM: %f, LPIPS: %f\n" %
(losses['total'].avg, psnrs.avg, ssims.avg, lpips.avg))
# Save psnr & ssim
save_fn = os.path.join('checkpoint', args.exp_name, save_folder, 'results.txt')
with open(save_fn, 'a') as f:
f.write("PSNR: %f, SSIM: %f, LPIPS: %f\n" %
(psnrs.avg, ssims.avg, lpips.avg))
# Log to TensorBoard
if args.mode != 'test':
utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg,
optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i, mode='test')
return losses['total'].avg, psnrs.avg, ssims.avg, lpips.avg
def test():
df_column = ['Name']
df_column.extend([str(i) for i in range(1, seq_len + 1)])
df = pd.DataFrame(columns=df_column)
psnr_array = np.zeros((0, seq_len))
ssim_array = np.zeros((0, seq_len))
tqdm_loader = tqdm.tqdm(validationloader, ncols=80)
imgsave_folder = os.path.join(args.checkpoint_dir, 'Saved_imgs')
if not os.path.exists(imgsave_folder):
os.mkdir(imgsave_folder)
with torch.no_grad():
for validationIndex, (validationData, validationFrameIndex,
validationFile) in enumerate(tqdm_loader):
blurred_img = torch.zeros_like(validationData[0])
for image in validationData:
blurred_img += image
blurred_img /= len(validationData)
blurred_img = blurred_img.to(device)
batch_size = blurred_img.shape[0]
blurred_img = meanshift(blurred_img, mean, std, device, False)
c = center_estimation(blurred_img)
start, end = border_estimation(blurred_img, c)
start = meanshift(start, mean, std, device, True)
end = meanshift(end, mean, std, device, True)
blurred_img = meanshift(blurred_img, mean, std, device, True)
frame0 = validationData[0].to(device)
frame1 = validationData[-1].to(device)
batch_size = blurred_img.shape[0]
parallel = torch.mean(compare_ftn(start, frame0) +
compare_ftn(end, frame1),
dim=(1, 2, 3))
cross = torch.mean(compare_ftn(start, frame1) +
compare_ftn(end, frame0),
dim=(1, 2, 3))
I0 = torch.zeros_like(blurred_img)
I1 = torch.zeros_like(blurred_img)
for b in range(batch_size):
if parallel[b] <= cross[b]:
I0[b], I1[b] = start[b], end[b]
else:
I0[b], I1[b] = end[b], start[b]
psnrs = np.zeros((batch_size, seq_len))
ssims = np.zeros((batch_size, seq_len))
for vindex in range(seq_len):
frameT = validationData[vindex]
IFrame = frameT.to(device)
if vindex == 0:
Ft_p = I0.clone()
elif vindex == seq_len - 1:
Ft_p = I1.clone()
else:
validationIndex = torch.ones(batch_size) * (vindex - 1)
validationIndex = validationIndex.long()
flowOut = flowComp(torch.cat((I0, I1), dim=1))
F_0_1 = flowOut[:, :2, :, :]
F_1_0 = flowOut[:, 2:, :, :]
fCoeff = superslomo.getFlowCoeff(validationIndex, device,
seq_len)
F_t_0 = fCoeff[0] * F_0_1 + fCoeff[1] * F_1_0
F_t_1 = fCoeff[2] * F_0_1 + fCoeff[3] * F_1_0
g_I0_F_t_0 = validationFlowBackWarp(I0, F_t_0)
g_I1_F_t_1 = validationFlowBackWarp(I1, F_t_1)
if args.add_blur:
intrpOut = ArbTimeFlowIntrp(
torch.cat((I0, I1, F_0_1, F_1_0, F_t_1, F_t_0,
g_I1_F_t_1, g_I0_F_t_0, blurred_img),
dim=1))
else:
intrpOut = ArbTimeFlowIntrp(
torch.cat((I0, I1, F_0_1, F_1_0, F_t_1, F_t_0,
g_I1_F_t_1, g_I0_F_t_0),
dim=1))
F_t_0_f = intrpOut[:, :2, :, :] + F_t_0
F_t_1_f = intrpOut[:, 2:4, :, :] + F_t_1
V_t_0 = torch.sigmoid(intrpOut[:, 4:5, :, :])
V_t_1 = 1 - V_t_0
g_I0_F_t_0_f = validationFlowBackWarp(I0, F_t_0_f)
g_I1_F_t_1_f = validationFlowBackWarp(I1, F_t_1_f)
wCoeff = superslomo.getWarpCoeff(validationIndex, device,
seq_len)
Ft_p = (wCoeff[0] * V_t_0 * g_I0_F_t_0_f +
wCoeff[1] * V_t_1 * g_I1_F_t_1_f) / (
wCoeff[0] * V_t_0 + wCoeff[1] * V_t_1)
Ft_p = meanshift(Ft_p, mean, std, device, False)
IFrame = meanshift(IFrame, mean, std, device, False)
for b in range(batch_size):
foldername = os.path.basename(
os.path.dirname(validationFile[ctr_idx][b]))
filename = os.path.splitext(
os.path.basename(validationFile[vindex][b]))[0]
out_fname = foldername + '_' + filename + '_out.png'
gt_fname = foldername + '_' + filename + '.png'
out, gt = quantize(Ft_p[b]), quantize(IFrame[b])
# Comment two lines below if you want to save images
# torchvision.utils.save_image(out, os.path.join(imgsave_folder, out_fname), normalize=True, range=(0,255))
# torchvision.utils.save_image(gt, os.path.join(imgsave_folder, gt_fname), normalize=True, range=(0,255))
psnr, ssim = eval_metrics(Ft_p, IFrame)
psnrs[:, vindex] = psnr.cpu().numpy()
ssims[:, vindex] = ssim.cpu().numpy()
for b in range(batch_size):
rows = [validationFile[ctr_idx][b]]
rows.extend(list(psnrs[b]))
df = df.append(pd.Series(rows, index=df.columns),
ignore_index=True)
df.to_csv('{}/results_PSNR.csv'.format(args.checkpoint_dir))
def eval_model(model, data, params):
model.eval()
reference, candidate, source, alignments = [], [], [], []
count, total_count = 0, len(data['validset'])
validloader = data['validloader']
tgt_vocab = data['tgt_vocab']
for src, tgt, src_len, tgt_len, original_src, original_tgt in validloader:
if config.use_cuda:
src = src.cuda()
src_len = src_len.cuda()
with torch.no_grad():
if config.beam_size > 1:
samples, alignment, weight = model.beam_sample(
src, src_len, beam_size=config.beam_size, eval_=True)
else:
samples, alignment = model.sample(src, src_len)
candidate += [tgt_vocab.convertToLabels(s, utils.EOS) for s in samples]
source += original_src
reference += original_tgt
if alignment is not None:
alignments += [align for align in alignment]
count += len(original_src)
utils.progress_bar(count, total_count)
if config.unk and config.attention != 'None':
cands = []
for s, c, align in zip(source, candidate, alignments):
cand = []
for word, idx in zip(c, align):
if word == utils.UNK_WORD and idx < len(s):
try:
cand.append(s[idx])
except:
cand.append(word)
print("%d %d\n" % (len(s), idx))
else:
cand.append(word)
cands.append(cand)
if len(cand) == 0:
print('Error!')
candidate = cands
with codecs.open(params['log_path'] + 'candidate.txt', 'w+', 'utf-8') as f:
for i in range(len(candidate)):
f.write(" ".join(candidate[i]) + '\n')
results = utils.eval_metrics(reference, candidate, label_dict,
params['log_path'])
score = {}
result_line = ""
for metric in config.metrics:
score[metric] = results[metric]
result_line += metric + ": %s " % str(score[metric])
result_line += '\n'
params['log'](result_line)
return score
def _train_epoch(self, epoch):
self.logger.info('\n')
self.model.train()
if self.config['model']['args']['freeze_bn']:
if isinstance(self.model, torch.nn.DataParallel):
self.model.module.freeze_bn()
else:
self.model.freeze_bn()
self.wrt_mode = 'train'
tic = time.time()
tbar = tqdm(self.train_loader, ncols=130)
self._reset_metrics()
for batch_idx, sample in enumerate(tbar):
torch.cuda.synchronize(device=self.device)
self.data_time.update(time.time() - tic)
self.optimizer.zero_grad()
if self.prefetch:
data = sample['img']
target = sample['label']
else:
data = sample['img'].to(self.device)
target = sample['label'].to(self.device)
if self.config['model']['name'] == 'UNet_ConvLSTM':
output, _ = self.model(data)
else:
output = self.model(data)
# print(output.size(), target.size())
assert output.size()[2:] == target.size()[1:]
assert output.size()[1] == self.num_classes
loss = self.loss(output, target)
if isinstance(self.loss, torch.nn.DataParallel):
loss = loss.mean()
loss.backward()
self.optimizer.step()
self.total_loss.update(loss.item())
self.lr_scheduler.step(epoch=epoch - 2)
# measure time
torch.cuda.synchronize(device=self.device)
self.batch_time.update(time.time() - tic)
tic = time.time()
if batch_idx % self.log_step == 0:
self.wrt_step = (epoch - 1) * len(
self.train_loader) + batch_idx
self.writer.add_scalar('{}/loss'.format(self.wrt_mode),
loss.item(), self.wrt_step)
# FOR EVAL
seg_metrics = eval_metrics(output,
target,
self.num_classes,
is_train=True)
self._update_seg_metrics(*seg_metrics)
pixAcc, mIoU, _ = self._get_seg_metrics().values()
# PRINT INFO
tbar.set_description(
'TRAIN ({}) | Loss: {} | Acc: {:.4f} mIoU: {:.4f}'.format(
epoch, self.total_loss.average, pixAcc, mIoU))
# METRICS TO TENSORBOARD
seg_metrics = self._get_seg_metrics()
for k, v in list(seg_metrics.items())[:-1]:
self.writer.add_scalar('{}/{}'.format(self.wrt_mode, k), v,
self.wrt_step)
for i, opt_group in enumerate(self.optimizer.param_groups):
self.writer.add_scalar(
'{}/Learning_rate_{}'.format(self.wrt_mode, i),
opt_group['lr'], self.wrt_step)
# self.writer.add_scalar(f'{self.wrt_mode}/Momentum_{k}', opt_group['momentum'], self.wrt_step)
# RETURN LOSS & METRICS
log = {'loss': self.total_loss.average, **seg_metrics}
# if self.lr_scheduler is not None: self.lr_scheduler.step()
return log
def _valid_epoch(self, epoch):
if self.val_loader is None:
self.logger.warning(
'Not data loader was passed for the validation step, No validation is performed !'
)
return {}
self.logger.info('\n###### EVALUATION ######')
self.model.eval()
self.wrt_mode = 'val'
self._reset_metrics()
tbar = tqdm(self.val_loader, ncols=130)
with torch.no_grad():
val_visual = []
for batch_idx, sample in enumerate(tbar):
if self.prefetch:
data = sample['img']
target = sample['label']
else:
data = sample['img'].to(self.device)
target = sample['label'].to(self.device)
if self.config['model']['name'] == 'UNet_ConvLSTM':
output, _ = self.model(data)
else:
output = self.model(data)
assert output.size()[2:] == target.size()[1:]
assert output.size()[1] == self.num_classes
loss = self.loss(output, target)
if isinstance(self.loss, torch.nn.DataParallel):
loss = loss.mean()
self.total_loss.update(loss.item())
seg_metrics = eval_metrics(output,
target,
self.num_classes,
is_train=True)
# seg_metrics = seg_metrics[:4]
self._update_seg_metrics(*seg_metrics)
# LIST OF IMAGE TO VIZ (15 images)
if len(val_visual) < 15:
target_np = target.data.cpu().numpy()
output_np = output.data.max(1)[1].cpu().numpy()
if self.config['dataset']['name'] == "SeqLane":
val_visual.append([
sample['img'][0][-1].data.cpu(), target_np[0],
output_np[0]
])
else:
val_visual.append([
sample['img'][0].data.cpu(), target_np[0],
output_np[0]
])
# PRINT INFO
pixAcc, mIoU, _ = self._get_seg_metrics().values()
tbar.set_description(
'EVAL ({}) | Loss: {:.3f}, PixelAcc: {:.2f}, Mean IoU: {:.2f} |'
.format(epoch, self.total_loss.average, pixAcc, mIoU))
# WRTING & VISUALIZING THE MASKS
val_img = get_val_image(val_visual, self.MEAN, self.STD,
self.val_loader.dataset.palette)
self.writer.add_image(
'{}/inputs_targets_predictions'.format(self.wrt_mode), val_img,
self.wrt_step)
# METRICS TO TENSORBOARD
self.wrt_step = (epoch) * len(self.val_loader)
self.writer.add_scalar('{}/loss'.format(self.wrt_mode),
self.total_loss.average, self.wrt_step)
seg_metrics = self._get_seg_metrics()
for k, v in list(seg_metrics.items())[:-1]:
self.writer.add_scalar('{}/{}'.format(self.wrt_mode, k), v,
self.wrt_step)
log = {'val_loss': self.total_loss.average, **seg_metrics}
return log
for batch_idx, batch in enumerate(_trange):
batch = [b.to("cuda") for b in batch]
data, target = batch[:-1], batch[-1]
# clear the gradients of all optimized variables
optimizer.zero_grad()
# forward pass: compute predicted outputs by passing inputs to the model
output = model(data)
# calculate the loss
loss = loss_fn.forward(output, target)
loss.backward()
# perform a single optimization step (parameter update)
optimizer.step()
# update running training loss
train_loss += loss.item()
total_metrics += eval_metrics(output, target, metrics, writer)
if batch_idx % int(np.sqrt(dataloader.batch_size)) == 0:
_str = 'Train Epoch: {} Loss: {:.6f}'.format(epoch,loss.item())
_trange.set_description(_str)
# print training statistics
# calculate average loss over an epoch
# Add epoch metrics
loss = train_loss / len(dataloader)
metric_epoch = (total_metrics / len(dataloader)).tolist()
writer.add_scalar('loss', loss)
for i, metric in enumerate(metrics):
writer.add_scalar("%s"%metric.__name__, metric_epoch[i])
def eval_detector(args):
load_model_dir = args.load_model_dir
load_model_name = args.load_model_name
normal_data_dir = args.normal_data_dir
normal_data_name_train = args.normal_data_name_train
normal_data_name_test = args.normal_data_name_test
abnormal_data_dir = args.abnormal_data_dir
abnormal_data_name = args.abnormal_data_name
Pvalue_th = args.Pvalue_th
val_and_ref_name = args.normal_data_name_val_and_ref
gpu = args.gpu
AnomalyDetector = torch.load(load_model_dir + load_model_name)
AnomalyDetector.eval()
AnomalyDetector.th = Pvalue_th
if args.dummydata:
_, _, testing_normal_data = loaddata.load_normal_dummydata()
else:
if args.debug:
_, training_normal_data, _, = (loaddata.load_data_split(
data_dir=normal_data_dir,
file_name=normal_data_name_train,
split=(0.1, 0.8, 0.1)))
_, ref_normal_data, val_normal_data = loaddata.load_data_split(
data_dir=normal_data_dir,
file_name=val_and_ref_name,
split=(0.1, 0.45, 0.45))
training_normal_data = torch.tensor(
AnomalyDetector.normalize(training_normal_data))
val_normal_data = torch.tensor(
AnomalyDetector.normalize(val_normal_data))
ref_normal_data = torch.tensor(
AnomalyDetector.normalize(ref_normal_data))
testing_normal_data = loaddata.load_data_all(
data_dir=normal_data_dir, file_name=normal_data_name_test)
testing_normal_data = torch.tensor(
AnomalyDetector.normalize(testing_normal_data))
if args.dummydata:
testing_abnormal_data = loaddata.load_abnormal_dummydata()
else:
testing_abnormal_data = loaddata.load_data_all(
data_dir=abnormal_data_dir, file_name=abnormal_data_name)
testing_abnormal_data = torch.tensor(
AnomalyDetector.normalize(testing_abnormal_data))
print("testing_abnormal_data.shape ", testing_abnormal_data.shape)
if gpu:
AnomalyDetector = AnomalyDetector.cuda()
testing_normal_data = testing_normal_data.cuda()
testing_abnormal_data = testing_abnormal_data.cuda()
true_label_normal = np.zeros(
len(testing_normal_data) -
AnomalyDetector.RED_collection_len * AnomalyDetector.RED_points - 1)
true_label_abnormal = np.ones(
len(testing_abnormal_data) -
AnomalyDetector.RED_collection_len * AnomalyDetector.RED_points - 1)
true_label = np.concatenate((true_label_normal, true_label_abnormal),
axis=0)
pred_normal, p_values_normal = AnomalyDetector.predict(testing_normal_data,
gpu,
debug=args.debug)
if args.debug:
feature_idx = 0
# debug_pred_normal is of size [seq_len-1, batch(=1), features]
RE_normal, debug_pred_normal = AnomalyDetector._get_reconstruction_error(
testing_normal_data, gpu=gpu)
seq_dict = {
"truth": testing_normal_data[1:, feature_idx].detach().numpy(),
"pred": debug_pred_normal[:, 0, feature_idx].detach().numpy(),
}
seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2
utils.plot_seq(seq_dict, title="Testing normal prediction")
# debug_pred_normal is of size [seq_len-1, batch(=1), features]
RE_abnormal, debug_pred_abnormal = AnomalyDetector._get_reconstruction_error(
testing_abnormal_data, gpu=gpu)
seq_dict = {
"truth": testing_abnormal_data[1:, feature_idx].detach().numpy(),
"pred": debug_pred_abnormal[:, 0, feature_idx].detach().numpy(),
}
seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2
utils.plot_seq(seq_dict, title="Testing abnormal prediction")
# debug_ref is of size [seq_len-1, batch(=1), features]
RE_ref, debug_ref = AnomalyDetector._get_reconstruction_error(
ref_normal_data, gpu=gpu)
seq_dict = {
"truth": ref_normal_data[1:, feature_idx].detach().numpy(),
"pred": debug_ref[:, 0, feature_idx].detach().numpy(),
}
seq_dict["diff"] = (seq_dict["pred"] - seq_dict["truth"])**2
utils.plot_seq(seq_dict, title="Train normal ref prediction")
RE_seq_dict = {
"RE_reference": RE_ref,
"RE_normal": RE_normal,
"RE_abnormal": RE_abnormal
}
utils.plot_seq(RE_seq_dict, title="Reconstruction errors")
utils.plot_cdf(RE_seq_dict, title="RED cdf")
print("p_values_normal.shape ", len(p_values_normal))
print("p_values_normal.mean ", np.mean(p_values_normal))
pred_abnormal, p_values_abnormal = AnomalyDetector.predict(
testing_abnormal_data, gpu, debug=args.debug)
print("p_values_abnormal.shape ", len(p_values_abnormal))
print("p_values_abnormal.mean ", np.mean(p_values_abnormal))
pred = np.concatenate((pred_normal, pred_abnormal), axis=0)
pred_score = np.concatenate((p_values_normal, p_values_abnormal), axis=0)
print("true_label.shape", true_label.shape, "pred.shape", pred.shape)
tp, fp, fn, tn, acc, prec, rec, f1, fpr, tpr, thresholds, roc_auc = (
utils.eval_metrics(
truth=true_label,
pred=pred,
anomaly_score=-np.log10(
pred_score + 1e-500) # Anomaly score=-log(p_value)
))
plt.figure()
plt.plot(fpr, tpr, color='darkorange')
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
plt.xlim([-0.01, 1.01])
plt.ylim([-0.01, 1.01])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC of LSTM anomaly detector')
plt.legend(loc="lower right")
plt.show()
return roc_auc
def run_train(det_save_path, n_classes, from_tfr):
graph = tf.get_default_graph()
sess = tf.Session(graph=graph)
if from_tfr:
C_dataset = TFR_Dataset(FLAGS.epochs,
FLAGS.batch_size,
FLAGS.prefetch_num,
TRAIN_ROOT,
10000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
file_shards=FLAGS.train_shards,
augment=True,
preprocess=True)
C_dataset.make_tfr_dataset()
else:
C_dataset = Memory_Dataset(FLAGS.epochs,
FLAGS.batch_size,
FLAGS.prefetch_num,
UP_TRAIN_PRECROP,
2000,
PRECROP_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
preprocess=True)
C_dataset.make_memory_dataset(aug_crop=True)
fluid_iterator = Dataset_iterator(C_dataset.dataset)
el = fluid_iterator.get_el()
net = Recognition_Network(n_classes,
FLAGS.box_gamma,
FLAGS.alpha_tail,
sess,
graph,
P_VALUE,
alpha_head=FLAGS.alpha_head)
net.set_params(el)
net.build_optimizer(lr=FLAGS.learning_rate,
resnet_path=RESNET_MODEL,
patch_model_path=PATCH_MODEL,
total_model_path=TOTAL_MODEL)
step = 0
if from_tfr:
net.sess.run(
fluid_iterator.return_initialized_iterator(C_dataset.dataset))
else:
C_dataset.make_memory_feed_dict()
net.sess.run(fluid_iterator.return_initialized_iterator(
C_dataset.dataset),
feed_dict=C_dataset.feed_dict)
train_start_time = time.time()
interval_time = train_start_time
try:
while True:
_, losses = net.sess.run([net.train_op, net.losses])
if step % FLAGS.print_step == 0: # ~46000 cases / batch size of 6 ~= 7.7e3 steps per epoch
# ~ 1800 / 6 ~= 300 steps per epoch for UP cases
c_end = time.time()
from_start = (c_end - train_start_time
) / 3600 # minutes and seconds to hours
from_last_step = (c_end -
interval_time) / 60 # seconds to hours
print(
'Step {0:g}, error {1:.3f}, {2:.2f} hrs from start, {3:.2f} mins from last epoch'
.format(step, losses['total'], from_start, from_last_step))
interval_time = c_end
step += 1
if step % FLAGS.save_step == 0: # 7700 steps per epochs * 40 epochs = 3.0e5 steps
# 300 steps/epoch * 2000 epochs for UP cases = 4.5e5 steps, just save only at end
net.total_model_saver.save(net.sess,
det_save_path,
global_step=step)
except tf.errors.OutOfRangeError:
net.total_model_saver.save(net.sess, det_save_path, global_step=step)
final_timing = (time.time() - train_start_time) / 3600
print('Done training after total time of {0:.1f} hours'.format(
final_timing))
print('L2 loss tensors are: {}'.format(
graph.get_collection('l2_loss')))
# Scoring; no longer in training with BN
if from_tfr:
# need to reset C_dataset to change epoch
C_dataset = TFR_Dataset(1,
FLAGS.infer_batch,
FLAGS.prefetch_num,
TRAIN_ROOT,
10000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
file_shards=FLAGS.train_shards,
augment=False,
preprocess=False)
C_dataset.make_tfr_dataset()
net.sess.run(
fluid_iterator.return_initialized_iterator(C_dataset.dataset))
else:
C_dataset = Memory_Dataset(1,
FLAGS.infer_batch,
FLAGS.prefetch_num,
UP_TRAIN_300,
2000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
preprocess=False)
C_dataset.make_memory_dataset()
C_dataset.make_memory_feed_dict()
net.sess.run(fluid_iterator.return_initialized_iterator(
C_dataset.dataset),
feed_dict=C_dataset.feed_dict)
pred_dict, labels_list = create_holders(n_classes)
while True:
try:
scores, singles = net.score()
for i, j in scores.items():
pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0)
labels_list = np.concatenate([labels_list, singles], axis=0)
except tf.errors.OutOfRangeError:
break
print('The training metrics are: ')
eval_metrics(pred_dict, labels_list)
#Scoring for test set;
if from_tfr:
C_dataset = TFR_Dataset(1,
FLAGS.infer_batch,
FLAGS.prefetch_num,
TEST_ROOT,
10000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
file_shards=FLAGS.train_shards,
augment=False,
preprocess=False)
C_dataset.make_tfr_dataset()
net.sess.run(
fluid_iterator.return_initialized_iterator(C_dataset.dataset))
else:
C_dataset = Memory_Dataset(1,
FLAGS.infer_batch,
FLAGS.prefetch_num,
UP_TEST,
2000,
TARGET_IMG_SIZE,
TARGET_IMG_SIZE,
LABEL_SIZE,
SINGLES,
preprocess=False)
C_dataset.make_memory_dataset()
C_dataset.make_memory_feed_dict()
net.sess.run(fluid_iterator.return_initialized_iterator(
C_dataset.dataset),
feed_dict=C_dataset.feed_dict)
pred_dict, labels_list = create_holders(n_classes)
while True:
try:
scores, singles = net.score()
for i, j in scores.items():
pred_dict[i] = np.concatenate([pred_dict[i], j], axis=0)
labels_list = np.concatenate([labels_list, singles], axis=0)
except tf.errors.OutOfRangeError:
break
print('The Testing metrics are: ')
eval_metrics(pred_dict, labels_list)
def train(args, epoch):
### GUI things
global psnrs
global out
global gt
global it
global apptr
### progres bar
global startedpbar
startedpbar=0
### loss
global LOSS_0
losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
model.train()
criterion.train()
t = time.time()
for i, (images, imgpaths) in enumerate(train_loader):
#print(startedpbar)
if startedpbar==0:
pbar=tqdm(range(i, len(train_loader)))
startedpbar=1
else:
startedpbar=1
#print(startedpbar)
# Build input batch
im1, im2, gt = utils.build_input(images, imgpaths)
# Forward
optimizer.zero_grad()
out, feats = model(im1, im2)
it+=1
loss, loss_specific = criterion(out, gt, None, feats)
QApplication.processEvents()
# Save loss values
for k, v in losses.items():
if k != 'total':
v.update(loss_specific[k].item())
if LOSS_0 == 0:
LOSS_0 = loss.data.item()
losses['total'].update(loss.item())
# Backward (+ grad clip) - if loss explodes, skip current iteration
loss.backward()
if loss.data.item() > 10.0 * LOSS_0:
print(max(p.grad.data.abs().max() for p in model.parameters()))
continue
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
optimizer.step()
# Calc metrics & print logs
if i % args.log_iter == 0:
utils.eval_metrics(out, gt, psnrs, ssims, lpips, lpips_model)
pbar.update(1)
pbar.set_postfix({'psnr': psnrs.avg, 'Loss': losses['total'].avg, 'epoch': epoch, 'iterations': it })
# Log to TensorBoard
utils.log_tensorboard(writer, losses, psnrs.avg, ssims.avg, lpips.avg,
optimizer.param_groups[-1]['lr'], epoch * len(train_loader) + i)
# Reset metrics
t = time.time()
# update gui
if args.gui=="True":
tgui = threading.Thread(target=updategui)
tgui.start()
losses, psnrs, ssims, lpips = utils.init_meters(args.loss)
def validate(model, epoch):
val_dataset = VimeoSeptuplet(
cfg
) #('data/vimeo_septuplet', [1, 2, 3, 4, 5, 6, 7], is_training=False)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=cfg.batch_size,
shuffle=False,
num_workers=8,
pin_memory=True)
#model.eval()
if True: #with torch.no_grad():
losses = utils.AverageMeter()
psnrs = utils.AverageMeter()
ssims = utils.AverageMeter()
batch_time = utils.AverageMeter()
t = time.time()
for i, (images, metadata) in enumerate(tqdm(val_loader)):
if i == VAL_ITER_CUT:
break
t = time.time()
images = [im.cuda() for im in images]
target = images[3]
weights_before = copy.deepcopy(net.state_dict())
val_optimizer = torch.optim.Adamax(net.parameters(),
lr=cfg.inner_lr)
# inner loop
k = cfg.num_inner_update
for _k in range(k):
indices = [[0, 2, 4], [2, 4, 6]]
total_loss = 0
for ind in indices:
output = net(images[ind[0]].clone(),
images[ind[2]].clone())
loss = criterion(output, images[ind[1]])
total_loss = total_loss + loss
val_optimizer.zero_grad()
total_loss.backward()
val_optimizer.step()
with torch.no_grad():
input0, input1, target = images[2], images[4], images[3]
output = model(input0, input1)
batch_time.update(time.time() - t)
loss = criterion(output, target)
losses.update(loss.item(), input0.size(0))
utils.eval_metrics(output, target, psnrs, ssims, None)
print(psnrs.val, ssims.val)
if VIZ:
for b in range(images[0].size(0)):
imgpath = metadata['imgpaths'][0][b]
#print(imgpath)
savepath = os.path.join('checkpoint', cfg.exp_name,
'vimeoSeptuplet',
imgpath.split('/')[-3],
imgpath.split('/')[-2])
if not os.path.exists(savepath):
#print('make dirs... %s' % savepath)
os.makedirs(savepath)
img_pred = (output[b].data.permute(1, 2, 0).clamp_(
0, 1).cpu().numpy()[..., ::-1] * 255).astype(
numpy.uint8)
cv2.imwrite(os.path.join(savepath, 'im2_pred.png'),
img_pred)
#batch_time.update(time.time() - t)
#t = time.time()
# restore the original base weight
net.load_state_dict(weights_before)
print("val_losses: %f" % losses.avg)
print("val_PSNR: %f, val_SSIM: %f" % (psnrs.avg, ssims.avg))
print("Time per batch: %.3f" % batch_time.avg)
return psnrs.avg
def train(epoch):
print("Training Epoch: %d" % epoch)
net.train()
losses = utils.AverageMeter()
psnrs = utils.AverageMeter()
ssims = utils.AverageMeter()
batch_time = utils.AverageMeter()
t = time.time()
for i, (images, _) in enumerate(train_loader):
if i == 4000:
break
images = [im.cuda() for im in images]
# Meta training
k = cfg.num_inner_update # inner update iteration
if cfg.meta_algorithm == 'reptile':
weights_before = copy.deepcopy(net.state_dict())
inner_optimizer = torch.optim.Adamax(net.parameters(),
lr=cfg.inner_lr)
# inner loop
for _k in range(k):
indices = [[0, 2, 4], [2, 4, 6]]
total_loss = 0
for ind in indices:
output = net(images[ind[0]].clone(),
images[ind[2]].clone())
loss = criterion(output, images[ind[1]])
total_loss = total_loss + loss
inner_optimizer.zero_grad()
total_loss.backward()
inner_optimizer.step()
# Reptile - outer update
outerstepsize = cfg.outer_lr
weights_after = net.state_dict()
net.load_state_dict({
name: weights_before[name] +
(weights_after[name] - weights_before[name]) * outerstepsize
for name in weights_before
})
# calculate loss w/ updated model
input0, input1, target = images[2], images[4], images[3]
with torch.no_grad():
output = net(input0, input1)
loss = criterion(output, target)
elif cfg.meta_algorithm == 'maml':
base_net = copy.deepcopy(net)
inner_optimizer = torch.optim.Adamax(net.parameters(),
lr=cfg.inner_lr)
# inner loop
for _k in range(k):
indices = [[0, 2, 4], [2, 4, 6]]
total_loss = 0
for ind in indices:
output = net(images[ind[0]].clone(),
images[ind[2]].clone())
loss = criterion(output, images[ind[1]])
total_loss = total_loss + loss
inner_optimizer.zero_grad()
total_loss.backward()
inner_optimizer.step()
# Forward on query data
outerstepsize = cfg.outer_lr
input0, input1, target = images[2], images[4], images[3]
output = net(input0, input1)
loss = criterion(output, target)
# Copy base parameters to 'net' to connect the computation graph
for param, base_param in zip(net.parameters(),
base_net.parameters()):
param.data = base_param.data
# Calculate gradient & update meta-learner
optimizer.zero_grad()
grads = torch.autograd.grad(loss, net.parameters())
for j, param in enumerate(net.parameters()):
#param = param - grads[j] * outerstepsize
param.grad = grads[j]
optimizer.step()
losses.update(loss.item(), images[0].size(0))
batch_time.update(time.time() - t)
t = time.time()
# Logging
if i % cfg.logfreq == 0:
utils.eval_metrics(output, target, psnrs, ssims, None)
print((
'Epoch: [%d][%d/%d],\tTime %.3f (%.3f)\tLoss %.4f (%.4f)\tPSNR %.2f\t'
% (epoch, i, len(train_loader), batch_time.val, batch_time.avg,
losses.val, losses.avg, psnrs.val)))
losses.reset()
psnrs.reset()
ssims.reset()
batch_time.reset()
