def test(self, restore_model, save_dir):
dataset = BasicDataset(data_list_file=self.dataset_config['data_list_file'], img_dir=self.dataset_config['img_dir'])
save_name_list = dataset.data_list[:, 2]
iterator = dataset.create_one_shot_iterator(dataset.data_list, num_parallel_calls=self.num_input_threads)
batch_img0, batch_img1, batch_img2 = iterator.get_next()
flow_fw, flow_bw = pyramid_processing(batch_img0, batch_img1, batch_img2, train=False, trainable=False, is_scale=True)
# For KITTI we set max_flow=256, while for Sintel we set max_flow=128
flow_fw_color = flow_to_color(flow_fw['full_res'], mask=None, max_flow=256)
flow_bw_color = flow_to_color(flow_bw['full_res'], mask=None, max_flow=256)
restore_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
saver = tf.train.Saver(var_list=restore_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer)
saver.restore(sess, restore_model)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for i in range(dataset.data_num):
np_flow_fw, np_flow_bw, np_flow_fw_color, np_flow_fw_color = sess.run([flow_fw['full_res'], flow_bw['full_res'], flow_fw_color, flow_bw_color])
misc.imsave('%s/flow_fw_color_%s.png' % (save_dir, save_name_list[i]), np_flow_fw_color[0])
misc.imsave('%s/flow_bw_color_%s.png' % (save_dir, save_name_list[i]), np_flow_bw_color[0])
write_flo('%s/flow_fw_%s.flo' % (save_dir, save_name_list[i]), np_flow_fw[0])
write_flo('%s/flow_bw_%s.flo' % (save_dir, save_name_list[i]), np_flow_bw[0])
print('Finish %d/%d' % (i, dataset.data_num))
def generate_fake_flow_occlusion(self, restore_model, save_dir):
dataset = BasicDataset(data_list_file=self.dataset_config['data_list_file'], img_dir=self.dataset_config['img_dir'])
save_name_list = dataset.data_list[:, 5]
iterator = dataset.create_one_shot_five_frame_iterator(dataset.data_list, num_parallel_calls=self.num_input_threads)
batch_img0, batch_img1, batch_img2, batch_img3, batch_img4 = iterator.get_next()
flow_fw_12, flow_bw_10, flow_fw_23, flow_bw_21, flow_fw_34, flow_bw_32 = pyramid_processing_five_frame(batch_img0, batch_img1, batch_img2, batch_img3, batch_img4,
train=False, trainable=False, regularizer=None, is_scale=True)
occ_fw_12, occ_bw_21 = occlusion(flow_fw_12['full_res'], flow_bw_21['full_res'])
occ_fw_23, occ_bw_32 = occlusion(flow_fw_23['full_res'], flow_bw_32['full_res'])
flow_fw_12_full_res = flow_fw_12['full_res'] * 64. + 32768
flow_occ_12_fw = tf.concat([flow_fw_12_full_res, occ_fw_12], -1)
flow_occ_12_fw = tf.cast(flow_occ_12_fw, tf.uint16)
flow_bw_21_full_res = flow_bw_21['full_res'] * 64. + 32768
flow_occ_21_bw = tf.concat([flow_bw_21_full_res, occ_bw_21], -1)
flow_occ_21_bw = tf.cast(flow_occ_21_bw, tf.uint16)
flow_fw_23_full_res = flow_fw_23['full_res'] * 64. + 32768
flow_occ_23_fw = tf.concat([flow_fw_23_full_res, occ_fw_23], -1)
flow_occ_23_fw = tf.cast(flow_occ_23_fw, tf.uint16)
flow_bw_32_full_res = flow_bw_32['full_res'] * 64. + 32768
flow_occ_32_bw = tf.concat([flow_bw_32_full_res, occ_bw_32], -1)
flow_occ_32_bw = tf.cast(flow_occ_32_bw, tf.uint16)
restore_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
saver = tf.train.Saver(var_list=restore_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer)
saver.restore(sess, restore_model)
#save_dir = '/'.join([self.save_dir, 'sample', self.model_name])
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for i in range(dataset.data_num):
flow_occ_12, flow_occ_21, flow_occ_23, flow_occ_32 = sess.run([flow_occ_12_fw, flow_occ_21_bw, flow_occ_23_fw, flow_occ_32_bw])
# opencv read and save image as bgr format, here we change rgb to bgr
np_flow_occ_12_fw = rgb_bgr(flow_occ_12[0])
np_flow_occ_21_bw = rgb_bgr(flow_occ_21[0])
np_flow_occ_12_fw = np_flow_occ_12_fw.astype(np.uint16)
np_flow_occ_21_bw = np_flow_occ_21_bw.astype(np.uint16)
np_flow_occ_23_fw = rgb_bgr(flow_occ_23[0])
np_flow_occ_32_bw = rgb_bgr(flow_occ_32[0])
np_flow_occ_23_fw = np_flow_occ_23_fw.astype(np.uint16)
np_flow_occ_32_bw = np_flow_occ_32_bw.astype(np.uint16)
cv2.imwrite('%s/flow_occ_12_fw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_12_fw)
cv2.imwrite('%s/flow_occ_21_bw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_21_bw)
cv2.imwrite('%s/flow_occ_23_fw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_23_fw)
cv2.imwrite('%s/flow_occ_32_bw_%s.png' % (save_dir, save_name_list[i]), np_flow_occ_32_bw)
print('Finish %d/%d' % (i, dataset.data_num))
def test(self, restore_model, save_dir, is_normalize_img=True, prefix=''):
dataset = BasicDataset(data_list_file=self.dataset_config['data_list_file'], img_dir=self.dataset_config['img_dir'] + prefix, is_normalize_img=is_normalize_img)
save_name_list = dataset.data_list[:, -1]
iterator = dataset.create_one_shot_iterator(dataset.data_list, num_parallel_calls=self.num_input_threads)
batch_img0, batch_img1, batch_img2 = iterator.get_next()
img_shape = tf.shape(batch_img0)
h = img_shape[1]
w = img_shape[2]
new_h = tf.where(tf.equal(tf.mod(h, 64), 0), h, (tf.to_int32(tf.floor(h / 64) + 1)) * 64)
new_w = tf.where(tf.equal(tf.mod(w, 64), 0), w, (tf.to_int32(tf.floor(w / 64) + 1)) * 64)
batch_img0 = tf.image.resize_images(batch_img0, [new_h, new_w], method=1, align_corners=True)
batch_img1 = tf.image.resize_images(batch_img1, [new_h, new_w], method=1, align_corners=True)
batch_img2 = tf.image.resize_images(batch_img2, [new_h, new_w], method=1, align_corners=True)
flow_fw, flow_bw = pyramid_processing(batch_img0, batch_img1, batch_img2, train=False, trainable=False, is_scale=True)
flow_fw['full_res'] = flow_resize(flow_fw['full_res'], [h, w], method=1)
flow_bw['full_res'] = flow_resize(flow_bw['full_res'], [h, w], method=1)
flow_fw_color = flow_to_color(flow_fw['full_res'], mask=None, max_flow=256)
flow_bw_color = flow_to_color(flow_bw['full_res'], mask=None, max_flow=256)
restore_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
saver = tf.train.Saver(var_list=restore_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer)
saver.restore(sess, restore_model)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for i in range(dataset.data_num):
np_flow_fw, np_flow_bw, np_flow_fw_color, np_flow_bw_color = sess.run([flow_fw['full_res'], flow_bw['full_res'], flow_fw_color, flow_bw_color])
misc.imsave(('%s/' + '%s.png') % (save_dir, save_name_list[i]), np_flow_fw_color[0])
# misc.imsave(('%s/' + prefix + '_%s.png') % (save_dir, save_name_list[i]), np_flow_fw_color[0])
# misc.imsave(('%s/' + prefix + '_flow_fw_color_%s.png') % (save_dir, save_name_list[i]), np_flow_fw_color[0])
# misc.imsave(('%s/' + prefix + '_flow_bw_color_%s.png') % (save_dir, save_name_list[i]), np_flow_bw_color[0])
# write_flo('%s/flow_fw_%s.flo' % (save_dir, save_name_list[i]), np_flow_fw[0])
# write_flo('%s/flow_bw_%s.flo' % (save_dir, save_name_list[i]), np_flow_bw[0])
print('Finish %d/%d' % (i+1, dataset.data_num))
def create_dataset_and_iterator(self, training_mode='no_self_supervision'):
if training_mode=='no_self_supervision':
dataset = BasicDataset(crop_h=self.dataset_config['crop_h'],
crop_w=self.dataset_config['crop_w'],
batch_size=self.batch_size_per_gpu,
data_list_file=self.dataset_config['data_list_file'],
img_dir=self.dataset_config['img_dir'])
iterator = dataset.create_batch_iterator(data_list=dataset.data_list, batch_size=dataset.batch_size,
shuffle=True, buffer_size=self.buffer_size, num_parallel_calls=self.num_input_threads)
elif training_mode == 'self_supervision':
dataset = BasicDataset(crop_h=self.dataset_config['crop_h'],
crop_w=self.dataset_config['crop_w'],
batch_size=self.batch_size_per_gpu,
data_list_file=self.dataset_config['data_list_file'],
img_dir=self.dataset_config['img_dir'],
fake_flow_occ_dir=self.self_supervision_config['fake_flow_occ_dir'],
superpixel_dir=self.dataset_config['superpixel_dir'])
iterator = dataset.create_batch_distillation_iterator(data_list=dataset.data_list, batch_size=dataset.batch_size,
shuffle=True, buffer_size=self.buffer_size, num_parallel_calls=self.num_input_threads)
else:
raise ValueError('Invalid training_mode. Training_mode should be one of {no_self_supervision, self_supervision}')
return dataset, iterator
def train_net(noise_fraction,
lr=1e-3,
momentum=0.9,
batch_size=128,
dir_img='ISIC_2019_Training_Input/',
save_cp=True,
dir_checkpoint='checkpoints/ISIC_2019_Training_Input/',
epochs=10):
train = BasicDataset(dir_img, noise_fraction, mode='train')
test = BasicDataset(dir_img, noise_fraction, mode='test')
val = BasicDataset(dir_img, noise_fraction, mode='val')
# n_test = int(len(dataset) * test_percent)
# n_train = len(dataset) - n_val
# train, test = random_split(dataset, [n_train, n_test])
data_loader = DataLoader(train, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)
test_loader = DataLoader(test, batch_size=batch_size, shuffle=False, num_workers=8, pin_memory=True)
val_loader = DataLoader(val, batch_size=5, shuffle=False, num_workers=8, pin_memory=True)
# data_loader = get_mnist_loader(hyperparameters['batch_size'], classes=[9, 4], proportion=0.995, mode="train")
# test_loader = get_mnist_loader(hyperparameters['batch_size'], classes=[9, 4], proportion=0.5, mode="test")
val_data, val_labels = next(iter(val_loader))
val_data = to_var(val_data, requires_grad=False)
val_labels = to_var(val_labels, requires_grad=False)
data = iter(data_loader)
net, opt = build_model(lr)
plot_step = 100
accuracy_log = []
logging.info(f'''Starting training:
Epochs: {epochs}
Batch size: {batch_size}
Learning rate: {lr}
Checkpoints: {save_cp}
Noise fraction: {noise_fraction}
Image dir: {dir_img}
''')
for epoch in range(epochs):
net.train()
for i in tqdm(range(int(len(train)/batch_size))):
# Line 2 get batch of data
image, labels = next(data)
try:
image, labels = next(data)
except StopIteration:
data = iter(data_loader)
image, labels = next(data)
# image, labels = next(iter(data_loader))
# since validation data is small I just fixed them instead of building an iterator
# initialize a dummy network for the meta learning of the weights
meta_net = model.resnet101(pretrained=False, num_classes=9)
meta_net.load_state_dict(net.state_dict())
if torch.cuda.is_available():
meta_net.cuda()
image = to_var(image, requires_grad=False)
labels = to_var(labels, requires_grad=False)
# Lines 4 - 5 initial forward pass to compute the initial weighted loss
# with torch.no_grad():
# print(image.shape)
y_f_hat = meta_net(image)
labels = labels.float()
cost = F.binary_cross_entropy_with_logits(y_f_hat, labels, reduce=False)
# print('cost:', cost)
eps = to_var(torch.zeros(cost.size()))
# print('eps: ', eps)
l_f_meta = torch.sum(cost * eps)
meta_net.zero_grad()
# Line 6 perform a parameter update
grads = torch.autograd.grad(l_f_meta, (meta_net.params()), create_graph=True, allow_unused=True)
meta_net.update_params(lr, source_params=grads)
# Line 8 - 10 2nd forward pass and getting the gradients with respect to epsilon
# with torch.no_grad():
y_g_hat = meta_net(val_data)
val_labels = val_labels.float()
l_g_meta = F.binary_cross_entropy_with_logits(y_g_hat, val_labels)
grad_eps = torch.autograd.grad(l_g_meta, eps, only_inputs=True)[0]
# Line 11 computing and normalizing the weights
w_tilde = torch.clamp(-grad_eps, min=0)
norm_c = torch.sum(w_tilde)
if norm_c != 0:
w = w_tilde / norm_c
else:
w = w_tilde
# Lines 12 - 14 computing for the loss with the computed weights
# and then perform a parameter update
# with torch.no_grad():
y_f_hat = net(image)
labels = labels.float()
cost = F.binary_cross_entropy_with_logits(y_f_hat, labels, reduce=False)
l_f = torch.sum(cost * w)
opt.zero_grad()
l_f.backward()
opt.step()
if i % plot_step == 0:
net.eval()
acc = []
for i, (test_img, test_label) in enumerate(test_loader):
test_img = to_var(test_img, requires_grad=False)
test_label = to_var(test_label, requires_grad=False)
with torch.no_grad():
output = net(test_img)
predicted = (F.sigmoid(output) > 0.5)
# print(type(predicted))
# predicted = to_var(predicted, requires_grad=False)
# print(type(predicted))
# test_label = test_label.float()
# print(type((predicted == test_label).float()))
acc.append((predicted.float() == test_label.float()).float())
accuracy = torch.cat(acc, dim=0).mean()
accuracy_log.append(np.array([i, accuracy])[None])
acc_log = np.concatenate(accuracy_log, axis=0)
if save_cp:
try:
os.mkdir(dir_checkpoint)
logging.info('Created checkpoint directory')
except OSError:
pass
torch.save(net.state_dict(),
dir_checkpoint + f'CP_epoch{epoch + 1}.pth')
logging.info(f'Checkpoint {epoch + 1} saved !')
# return accuracy
return np.mean(acc_log[-6:-1, 1])
def eval(self, restore_model, save_dir, is_normalize_img=True):
from test_datasets_eval import BasicDataset
from error_metrics import flow_error_avg, outlier_pct, merge_dictionaries
dataset = BasicDataset(data_list_file=self.dataset_config['data_list_file'], img_dir=self.dataset_config['img_dir'], is_normalize_img=is_normalize_img)
save_name_list = dataset.data_list[:, -1]
iterator = dataset.create_one_shot_iterator(dataset.data_list, num_parallel_calls=self.num_input_threads)
batch_img0, batch_img1, batch_img2, flow_noc, flow_occ, mask_noc, mask_occ = iterator.get_next()
img_shape = tf.shape(batch_img0)
h = img_shape[1]
w = img_shape[2]
new_h = tf.where(tf.equal(tf.mod(h, 64), 0), h, (tf.to_int32(tf.floor(h / 64) + 1)) * 64)
new_w = tf.where(tf.equal(tf.mod(w, 64), 0), w, (tf.to_int32(tf.floor(w / 64) + 1)) * 64)
batch_img0 = tf.image.resize_images(batch_img0, [new_h, new_w], method=1, align_corners=True)
batch_img1 = tf.image.resize_images(batch_img1, [new_h, new_w], method=1, align_corners=True)
batch_img2 = tf.image.resize_images(batch_img2, [new_h, new_w], method=1, align_corners=True)
flow_fw, _ = pyramid_processing(batch_img0, batch_img1, batch_img2, train=False, trainable=False, is_scale=True)
flow_fw['full_res'] = flow_resize(flow_fw['full_res'], [h, w], method=1)
flow_fw_color = flow_to_color(flow_fw['full_res'], mask=None, max_flow=256)
error_fw_color = flow_error_image(flow_fw['full_res'], flow_occ, mask_occ)
errors = {}
errors['EPE_noc'] = flow_error_avg(flow_noc, flow_fw['full_res'], mask_noc)
errors['EPE_all'] = flow_error_avg(flow_occ, flow_fw['full_res'], mask_occ)
errors['outliers_noc'] = outlier_pct(flow_noc, flow_fw['full_res'], mask_noc)
errors['outliers_all'] = outlier_pct(flow_occ, flow_fw['full_res'], mask_occ)
restore_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
saver = tf.train.Saver(var_list=restore_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer)
saver.restore(sess, restore_model)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
sum_EPE_noc = 0.
sum_EPE_all = 0.
sum_outliers_noc = 0.
sum_outliers_all = 0.
for i in range(dataset.data_num):
np_flow_fw, np_flow_fw_color, np_error_fw_color = sess.run([flow_fw['full_res'], flow_fw_color, error_fw_color])
EPE_noc, EPE_all, outliers_noc, outliers_all = sess.run([errors['EPE_noc'], errors['EPE_all'], errors['outliers_noc'], errors['outliers_all']])
sum_EPE_noc += EPE_noc
sum_EPE_all += EPE_all
sum_outliers_noc += outliers_noc
sum_outliers_all += outliers_all
misc.imsave('%s/%s_10.png' % (save_dir, save_name_list[i]), np_flow_fw_color[0])
misc.imsave('%s/error_%s.png' % (save_dir, save_name_list[i]), np_error_fw_color[0])
#write_flo('%s/flow_fw_%s.flo' % (save_dir, save_name_list[i]), np_flow_fw[0])
print('Finish %d/%d' % (i+1, dataset.data_num))
print("EPE_noc: %f, EPE_all: %f" % (sum_EPE_noc/dataset.data_num, sum_EPE_all/dataset.data_num))
print("F1_noc: %f, F1_all: %f" % (sum_outliers_noc/dataset.data_num, sum_outliers_all/dataset.data_num))
args = get_args()
net, opt = build_model(lr=args.lr)
net_losses = []
plot_step = 100
net_l = 0
smoothing_alpha = 0.9
accuracy_log = []
# data_loader = dl.get_mnist_loader(args.batch_size, classes=[9, 4], proportion=0.995, mode="train")
# test_loader = dl.get_mnist_loader(args.batch_size, classes=[9, 4], proportion=0.5, mode="test")
train = BasicDataset(imgs_dir=args.imgs_dir, mode='base')
test = BasicDataset(imgs_dir=args.imgs_dir, mode='test')
data_loader = DataLoader(train,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True)
test_loader = DataLoader(test,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True)
data = iter(data_loader)