def validate(model, dataset, epoch):
"""Perform an evaluation of `model` on the examples from `dataset`."""
batch_loss, avg_loss = 0., 0.
start = time.time()
for batch, (images, loc_trues,
cls_trues) in enumerate(tfe.Iterator(dataset)):
loc_preds, cls_preds = model(images)
loc_loss, cls_loss = loss_fn(loc_preds,
loc_trues,
cls_preds,
cls_trues,
num_classes=conf.num_class)
batch_loss += loc_loss + cls_loss
avg_loss = batch_loss / (batch + 1)
if batch % conf.log_interval == 0:
fmt = [
i.numpy()
for i in [loc_loss, cls_loss, loc_loss + cls_loss, avg_loss]
]
print(
"[EVALUATION] Batch: {}({:.0f}/{})\t".format(
batch, epoch, conf.num_epochs),
"loc_loss: {:.6f} | cls_loss: {:.6f} | total_loss: {:.6f} | avg_loss: {:.2f}"
.format(*fmt))
time_in_ms = (time.time() - start) / 60
print('[EVALUATION] Batch: {} Average time: {:.2f}min\n'.format(
batch, time_in_ms))
with tf.contrib.summary.always_record_summaries():
tf.contrib.summary.scalar('avg_loss', avg_loss)
return avg_loss
def train(self, mode=True):
train_loader = DataLoader(dataset=self.train_dataset,
batch_size=self.batch_size,
drop_last=True,
shuffle=False)
if len(self.train_dataset) == 0:
print("No training data was provided!")
return
generation = 0
for epoch in range(self.iteration):
print('\n\nTraining epoch: %d' % epoch)
self.scheduler.step()
print("Learning_rate: ", self.scheduler.get_lr()[0])
output3 = 0
label3 = 0
gse3 = 0
loss = 0
for items in train_loader:
generation += 1
img, label3, label2, label1, gse3, gse2 = self.to_cuda(*items)
self.optimizer.zero_grad()
output1, output2, output3 = self.model(img)
loss = loss_fn(output1, output2, output3, label1.long(),
label2.long(), label3.long(), gse2, gse3,
self.weight)
loss.backward()
self.optimizer.step()
output3 = np.argmax(output3.data.cpu().numpy(), 1)
label3 = np.squeeze(label3.data.cpu().numpy())
gse3 = gse3.data.cpu().numpy()
miou = self.IOU(output3, label3, self.num_class)
acc = np.equal(output3, label3)
gse_acc = np.sum(acc * gse3) / np.sum(gse3)
acc = np.mean(acc)
print('----------Loss = {:5.5f} ; Accuracy = {:2.2f}% ---------'.
format(np.float64(loss), acc * 100))
print('----------GSE_acc = {:2.2f}%; mIOU = {:2.2f}% ---------'.
format(gse_acc, miou))
cv2.imwrite('../pic3/{}.png'.format(epoch),
output3[0].reshape(512, 512) * 20)
if epoch % 10 == 0:
self.save(epoch)
def model_loss(images, loc_trues, cls_trues):
loc_preds, cls_preds = model(images, training=True)
loc_loss, cls_loss = loss_fn(loc_preds,
loc_trues,
cls_preds,
cls_trues,
num_classes=conf.num_class)
total_loss = loc_loss + cls_loss
tf.contrib.summary.scalar('loc_loss', loc_loss)
tf.contrib.summary.scalar('cls_loss', cls_loss)
tf.contrib.summary.scalar('total_loss', total_loss)
return total_loss
def validation(net: Model, loader: Loader):
val_accuracy = []
loss = []
queue_seg = Queue(maxsize=1)
queue_weightmap = Queue(maxsize=1)
p = Process(target=extract_weight_map, args=(queue_seg, queue_weightmap))
p.start()
for i, (image_batch,
seg_batch) in enumerate(loader.get_minibatch(train=False)):
queue_seg.put(seg_batch)
pred = net(image_batch)
weight_map = queue_weightmap.get()
loss.append(
loss_fn(seg=seg_batch, predictions=pred, weight_map=weight_map))
val_accuracy.append(Jaccard_Index(predictions=pred, seg=seg_batch))
if i == config.validation_steps:
return np.mean(val_accuracy), np.mean(loss)
p.terminate()
def train(net: Model, optimizer: tf.compat.v2.optimizers, loader: Loader):
if not os.path.exists(config.log_dir):
os.mkdir(config.log_dir)
queue_seg = Queue(maxsize=1)
queue_weightmap = Queue(maxsize=1)
p = Process(target=extract_weight_map, args=(queue_seg, queue_weightmap))
p.start()
Tensorcallback = callbacks.TensorBoard(config.log_dir,
write_graph=False,
write_images=False)
Tensorcallback.set_model(net)
for epoch in range(config.epochs):
train_accuracy = []
train_loss = []
for iteration, (image_batch,
seg_batch) in enumerate(loader.get_minibatch()):
queue_seg.put(seg_batch)
with tf.GradientTape() as tape:
pred = net(image_batch)
weight_map = queue_weightmap.get()
train_loss.append(
loss_fn(seg_batch, pred, weight_map=weight_map))
train_accuracy.append(Jaccard_Index(pred, seg_batch))
print("Epoch {}/{} iteration {} : loss {} accuracy {}".format(
epoch, config.epochs, iteration, train_loss[-1],
train_accuracy[-1]))
grads = tape.gradient(train_loss[-1], net.trainable_variables)
optimizer.apply_gradients(zip(grads, net.trainable_variables))
if iteration == config.steps_per_epoch:
break
train_loss_mean = np.mean(train_loss)
train_accuracy_mean = np.mean(train_accuracy)
if epoch % config.val_freq == 0: # output validation accuracy
val_accuracy, val_loss = validation(net, loader)
logs = {
'acc': train_accuracy_mean,
'loss': train_loss_mean,
'val_acc': val_accuracy,
'val_loss': val_loss
}
Tensorcallback.on_epoch_end(epoch=epoch, logs=logs)
Tensorcallback.on_train_end('_')
p.terminate()
def train_fn(data_loader, model, optimizer, device, scheduler):
model.train()
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets = d["targets"]
ids = ids.to(device, dtype=torch.long)
token_type_ids = token_type_ids.to(device, dtype=torch.long)
mask = mask.to(device, dtype=torch.long)
targets = targets.to(device, dtype=torch.float)
optimizer.zero_grad()
outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
loss = loss.loss_fn(outputs, targets)
loss.backward()
optimizer.step()
scheduler.step()
def __init__(self, model, device, config):
self.model = model
self.device = device
self.config = config
self.logger = logging.getLogger('training')
self.best_dice = 0
self.best_thres = None
self.epoch = 0
self.best_loss = np.inf
self.oof = None
self.monitored_metric = None
if not os.path.exists(self.config.SAVE_PATH):
os.makedirs(self.config.SAVE_PATH)
if not os.path.exists(self.config.LOG_PATH):
os.makedirs(self.config.LOG_PATH)
self.loss = loss_fn(config).to(self.device)
self.optimizer = getattr(torch.optim, config.optimizer)(self.model.parameters(),
**config.optimizer_params[config.optimizer])
self.scheduler = getattr(torch.optim.lr_scheduler, config.scheduler)(optimizer=self.optimizer,
**config.scheduler_params[config.scheduler])
n_batches // n_batch_verif), np.zeros(
n_batches // n_batch_verif), np.zeros(n_batches //
n_batch_verif)
vs, vf, tg = [], [], []
idx = -1
for batch_idx, (data, target, img_file,
class_id) in enumerate(train_loader):
data_set = data[np.arange(0, batch_size, n_samples)].to(device)
data_query = data[np.arange(1, batch_size,
n_samples)].to(device)
v_set = model(data_set, m=m_set) # single vector per set
v_f = model(data_query, m=1) # single vector per query
Sim = torch.mm(F.normalize(v_set, p=2, dim=1),
F.normalize(v_f, p=2, dim=1).t())
output = logisticReg(Sim.unsqueeze(-1)).squeeze()
loss_outputs, accuracy = loss_fn(output, len(v_f), m_set)
tot_acc += accuracy
tot_loss += loss_outputs
vs.append(v_set)
vf.append(v_f)
tg.append(target)
if (batch_idx + 1) % n_batch_verif == 0:
idx += 1
vs = torch.sign(
torch.stack(vs).flatten(start_dim=0, end_dim=1) +
1e-16)
vf = torch.sign(
torch.stack(vf).flatten(start_dim=0, end_dim=1) +
1e-16)
tg = torch.stack(tg).flatten(start_dim=0, end_dim=1)
def main():
# Training settings
parser = argparse.ArgumentParser(description='MultiView Training')
parser.add_argument('--root', type=str, default="kitti",
help='dataset root to load')
parser.add_argument('--num-inputs', type=int, default=4,
help='number of aux views')
parser.add_argument('--checkpoint', type=str, default=None,
help='checkpoint to start from')
parser.add_argument('--batch-size', type=int, default=8, metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=8, metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=1000000, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr-p', type=float, default=5e-5, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--lr-d', type=float, default=1e-4, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--lr-f', type=float, default=1e-4, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.9, metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--l2', type=float, default=5.0E-4)
parser.add_argument('--decay-frequency', type=int, default=500)
parser.add_argument('--deterministic', action='store_true',
help='fixed seed PRNG')
parser.add_argument('--seed', type=int, default=13, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--multi', action='store_true', default=False,
help='use all available GPUs')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
args = parser.parse_args()
device, dev_count = setup_devices(args.no_cuda, args.deterministic, args.seed)
if not args.multi:
dev_count = 1
args.batch_size *= dev_count
train_dataset, valid_dataset = create_default_splits(args.num_inputs, args.root, cache=args.multi)
viz = SummaryWriter()
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
drop_last=False)
valid_loader = DataLoader(valid_dataset,
batch_size=args.test_batch_size,
shuffle=False,
num_workers=1,
pin_memory=True)
model = Multi2Novel().to(device)
flow_params = []
pixel_params = []
pixel_gan_params = []
discr_params = []
for k, v in model.named_parameters():
if k.startswith('flow_predictor'):
flow_params.append(v)
elif k.startswith('pixel_generator'):
pixel_params.append(v)
pixel_gan_params.append(v)
elif k.startswith('discriminator'):
discr_params.append(v)
pixel_gan_params.append(v)
if args.multi:
model = nn.DataParallel(model)
opt_pixel = optim.Adam(pixel_params, lr=args.lr_p, weight_decay=args.l2)
opt_pixel_gan = optim.Adam(pixel_gan_params, lr=args.lr_p, weight_decay=args.l2)
opt_flow = optim.Adam(flow_params, lr=args.lr_f, weight_decay=args.l2)
opt_discr = optim.Adam(discr_params, lr=args.lr_d, weight_decay=args.l2)
it = 0
epoch_start = 1
best_loss = inf
if args.checkpoint is not None:
checkpoint_status = load_checkpoint(model, [opt_pixel, opt_pixel_gan, opt_flow, opt_discr], filename=args.checkpoint.split(".")[0])
if checkpoint_status is not None:
it, epoch_start, best_loss = checkpoint_status
batch_count = len(train_loader)
with tqdm.trange(epoch_start, args.epochs + 1, desc="epochs") as epoch_bar, tqdm.tqdm(
total=batch_count, leave=False, desc="train"
) as train_bar:
for epoch in epoch_bar:
avg_train_loss = 0.0
batch_it = 0.0
for batch in train_loader:
model.train()
opt_pixel.zero_grad()
opt_pixel_gan.zero_grad()
opt_flow.zero_grad()
opt_discr.zero_grad()
(inp_img, inp_pose), (target_img, target_pose) = batch
inputs = (inp_img.to(device), inp_pose.to(device))
targets = (target_img.to(device), target_pose.to(device))
outputs = model(inputs, targets)
px_loss, fl_loss, px_gan_loss, d_loss = loss_fn(*outputs, targets[0])
train_gan = it > 300000
train_loss = 0.0
opts = []
if train_gan:
if it % 2 > 0: # train generator
train_loss = px_gan_loss + fl_loss
opts += [opt_pixel_gan, opt_flow]
else: # train discriminator
train_loss = d_loss
opts += [opt_discr,]
else: # train pixel predictor + flow estimator
train_loss = px_loss + fl_loss
opts += [opt_pixel, opt_flow]
if viz:
viz.add_scalar('loss/training', train_loss, it)
train_loss.backward()
for opt in opts:
opt.step()
it += 1
train_bar.update()
train_bar.set_postfix(dict(total_it=it))
epoch_bar.refresh()
if (it % batch_count) == 0:
train_bar.close()
# validation
model.eval()
eval_dict = {}
val_loss = 0.0
for i, data in tqdm.tqdm(
enumerate(valid_loader, 0), total=len(valid_loader), leave=False, desc="val"
):
loss = None
acc = None
with torch.no_grad():
(inp_img, inp_pose), (target_img, target_pose) = data
inputs = (inp_img.to(device), inp_pose.to(device))
targets = (target_img.to(device), target_pose.to(device))
outputs = model(inputs, targets)
px_loss, fl_loss, _, _ = loss_fn(*outputs, targets[0])
per_batch_loss = px_loss + fl_loss
val_loss += (per_batch_loss - val_loss) / (i+1)
if viz:
viz.add_scalar('loss/validation', val_loss, it)
is_best = val_loss < best_loss
best_loss = min(best_loss, val_loss)
save_checkpoint(
checkpoint_state(
model, [opt_pixel, opt_pixel_gan, opt_flow, opt_discr], val_loss, epoch, it, False
),
is_best,
filename="checkpoints/step_ckpt",
bestname="checkpoints/best_ckpt",
)
train_bar = tqdm.tqdm(
total=batch_count, leave=False, desc="train"
)
train_bar.set_postfix(dict(total_it=it))
if viz:
viz.flush()
#
if viz is not None:
viz.close()
def build(self, reuse=False):
self.reuse = reuse
with tf.variable_scope(self.name, reuse=self.reuse):
# train set
self.mix = self.loader.mix_queue # mixture batch
self.source = self.loader.source_queue # source batch
self.other = self.mix - self.source
# test samples (placeholder)
self.mix_test = tf.placeholder(tf.float32, shape=(1,16384), name="mixture_sample")
self.source_test = tf.placeholder(tf.float32, shape=(1,16384), name="source_sample")
# define network
self.FE = FrontEnd()
if config.network == "unet":
self.SEP = Unet(model=config.model)
elif config.network == "lstm":
self.SEP = LSTM(model=config.model)
elif config.network == "bilstm":
self.SEP = BiLSTM(model=config.model)
elif config.network == "dilated":
self.SEP = Dilated(model=config.model)
else:
print("No model chosen")
raise ValueError
self.BE = BackEnd()
functions = [self.FE, self.SEP, self.BE]
if config.network2 == "unet":
self.SEP2 = Unet(model=config.model2)
self.CON = Confidence(model=config.confidence)
functions = functions + [self.SEP2, self.CON]
elif config.network2 == "lstm":
self.SEP2 = LSTM(model=config.model2)
self.CON = Confidence(model=config.confidence)
functions = functions + [self.SEP2, self.CON]
elif config.network2 == "bilstm":
self.SEP2 = BiLSTM(model=config.model2)
self.CON = Confidence(model=config.confidence)
functions = functions + [self.SEP2, self.CON]
elif config.network2 == "dilated":
self.SEP2 = Dilated(model=config.model2)
self.CON = Confidence(model=config.confidence)
functions = functions + [self.SEP2, self.CON]
else:
print("No model chosen")
input = {"mix": self.mix, "source": self.source}
input_test = {"mix": self.mix_test, "source": self.source_test}
# draw graph
self.graph = build_graph()
self.graph_test = build_graph(train=False)
self.graph(input, functions)
self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
self.graph_test(input_test, functions)
# variable count
self.variable_size = np.sum(
np.array([np.prod(np.array(v.get_shape().as_list())) for v in tf.trainable_variables()]))
print("\n total varialbes : {}".format(self.variable_size), "\n")
for v in tf.trainable_variables():
print(v, "{}".format(np.prod(np.array(v.get_shape().as_list()))))
# Define loss and summarize
# self.loss_pre = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
# self.source, self.other, loss_type=config.loss) + \
# loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
# self.source, self.other, loss_type=config.loss))/2
if config.hybrid:
if config.loss_seq == 'mid':
self.loss = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) + \
loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
elif config.loss_seq == 'end':
self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) + \
loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
elif config.loss_seq == 'both':
self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss))/2
elif config.loss_seq == 'two':
self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) + \
loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) +
loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)) / 2
elif config.loss_seq == 'first':
self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) + \
loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.graph.estimated = self.graph.estimated1
self.graph_test.estimated = self.graph_test.estimated1
elif config.loss_seq == 'second':
self.loss_pre = loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.loss = loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
self.source, self.other, loss_type=config.loss) + \
loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.graph.estimated = self.graph.estimated2
self.graph_test.estimated = self.graph_test.estimated2
else:
raise AssertionError("wrong config.loss_seq !!")
else:
self.loss = loss_fn(self.graph.masked_spec, self.graph.estimated, self.graph.source_spec,
self.source, self.other, loss_type=config.loss)
self.loss_summary = tf.summary.scalar("loss", self.loss)
self.alpha_summary = tf.summary.scalar("alpha", tf.reduce_mean(self.graph.alpha))
self.summary = tf.summary.merge_all()
print("\n loss type : %s \n" % config.loss)
epoch_time_start = time.time()
model.train()
logisticReg.train()
for batch_idx, value in enumerate(train_loader):
# data: (batch_size,3,224,224)
data = value[0]
target = value[1]
data_set = data[np.arange(0, batch_size, n_samples)].to(device)
data_query = data[np.arange(1, batch_size, n_samples)].to(device)
v_set, code_set = model(data_set, m=m_set) # single vector per set
v_f, code_f = model(data_query, m=1) # single vector per query
# Sim = torch.mm(code_set, code_f.t())
Sim = torch.mm(F.normalize(code_set, p=2, dim=1),
F.normalize(code_f, p=2, dim=1).t())
output = logisticReg(Sim.unsqueeze(-1)).squeeze()
loss1, accuracy = loss_fn(output, len(code_f), m_set)
h = torch.cat([v_set, v_f], dim=0)
loss2 = torch.mean(
torch.abs(
torch.pow(
torch.abs(h) - Variable(torch.ones(h.size()).cuda()),
3)))
loss_outputs = loss1 + alpha * loss2
tot_acc += accuracy
tot_loss += loss_outputs
optimizer_model.zero_grad()
# print('device:', loss_outputs.device, 'type:', loss_outputs.dtype, 'value:', loss_outputs)
loss_outputs.backward()
optimizer_model.step()
def train_epoch(self):
self.models.train()
self.epoch += 1
# record training statistics
avg_meters = {
'loss_rec': AverageMeter('Loss Rec', ':.4e'),
'loss_adv': AverageMeter('Loss Adv', ':.4e'),
'loss_disc': AverageMeter('Loss Disc', ':.4e'),
'time': AverageMeter('Time', ':6.3f')
}
progress_meter = ProgressMeter(len(self.train_loaders[0]),
avg_meters.values(),
prefix="Epoch: [{}]".format(self.epoch))
# begin training from minibatches
for ix, (data_0, data_1) in enumerate(zip(*self.train_loaders)):
start_time = time.time()
# load text and labels
src_0, src_len_0, labels_0 = data_0
src_0, labels_0 = src_0.to(args.device), labels_0.to(args.device)
src_1, src_len_1, labels_1 = data_1
src_1, labels_1 = src_1.to(args.device), labels_1.to(args.device)
# encode
encoder = self.models['encoder']
z_0 = encoder(labels_0, src_0, src_len_0) # (batch_size, dim_z)
z_1 = encoder(labels_1, src_1, src_len_1)
# recon & transfer
generator = self.models['generator']
inputs_0 = (z_0, labels_0, src_0)
h_ori_seq_0, pred_ori_0 = generator(*inputs_0, src_len_0, False)
h_trans_seq_0_to_1, _ = generator(*inputs_0, src_len_1, True)
inputs_1 = (z_1, labels_1, src_1)
h_ori_seq_1, pred_ori_1 = generator(*inputs_1, src_len_1, False)
h_trans_seq_1_to_0, _ = generator(*inputs_1, src_len_0, True)
# discriminate real and transfer
disc_0, disc_1 = self.models['disc_0'], self.models['disc_1']
d_0_real = disc_0(h_ori_seq_0.detach()) # detached
d_0_fake = disc_0(h_trans_seq_1_to_0.detach())
d_1_real = disc_1(h_ori_seq_1.detach())
d_1_fake = disc_1(h_trans_seq_0_to_1.detach())
# discriminator loss
loss_disc = (loss_fn(args.gan_type)(d_0_real, self.ones) +
loss_fn(args.gan_type)(d_0_fake, self.zeros) +
loss_fn(args.gan_type)(d_1_real, self.ones) +
loss_fn(args.gan_type)(d_1_fake, self.zeros))
# gradient penalty
if args.gan_type == 'wgan-gp':
loss_disc += args.gp_weight * gradient_penalty(
h_ori_seq_0, # real data for 0
h_trans_seq_1_to_0, # fake data for 0
disc_0)
loss_disc += args.gp_weight * gradient_penalty(
h_ori_seq_1, # real data for 1
h_trans_seq_0_to_1, # fake data for 1
disc_1)
avg_meters['loss_disc'].update(loss_disc.item(), src_0.size(0))
self.disc_optimizer.zero_grad()
loss_disc.backward()
self.disc_optimizer.step()
# reconstruction loss
loss_rec = (
F.cross_entropy( # Recon 0 -> 0
pred_ori_0.view(-1, pred_ori_0.size(-1)),
src_0[1:].view(-1),
ignore_index=bert_tokenizer.pad_token_id,
reduction='sum') + F.cross_entropy( # Recon 1 -> 1
pred_ori_1.view(-1, pred_ori_1.size(-1)),
src_1[1:].view(-1),
ignore_index=bert_tokenizer.pad_token_id,
reduction='sum')) / (
2.0 * args.batch_size
) # match scale with the orginal paper
avg_meters['loss_rec'].update(loss_rec.item(), src_0.size(0))
# generator loss
d_0_fake = disc_0(h_trans_seq_1_to_0) # not detached
d_1_fake = disc_1(h_trans_seq_0_to_1)
loss_adv = (loss_fn(args.gan_type, disc=False)
(d_0_fake, self.ones) +
loss_fn(args.gan_type, disc=False)(d_1_fake, self.ones)
) / 2.0 # match scale with the original paper
avg_meters['loss_adv'].update(loss_adv.item(), src_0.size(0))
# XXX: threshold for training stability
if (not args.two_stage):
if (args.threshold is not None and loss_disc < args.threshold):
loss = loss_rec + args.rho * loss_adv
else:
loss = loss_rec
else: # two_stage training
if (args.second_stage_num > args.epochs - self.epoch):
# last second_stage; flow loss_adv gradients
loss = loss_rec + args.rho * loss_adv
else:
loss = loss_rec
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
avg_meters['time'].update(time.time() - start_time)
# log progress
if (ix + 1) % args.log_interval == 0:
progress_meter.display(ix + 1)
progress_meter.display(len(self.train_loaders[0]))
n_batches = len(validation_loader)
Ptp01, Ptp05, Ptp1, AUC = np.zeros(n_batches // n_batch_verif), np.zeros(
n_batches // n_batch_verif), np.zeros(
n_batches // n_batch_verif), np.zeros(n_batches // n_batch_verif)
vs, vf, tg = [], [], []
idx = -1
with torch.no_grad():
for batch_idx, (data, target) in enumerate(validation_loader):
data_set = data[np.arange(0, batch_size, n_samples)].to(device)
data_query = data[np.arange(1, batch_size, n_samples)].to(device)
v_set, code_set = model(data_set, m=m_set) # single vector per set
v_f, code_f = model(data_query, m=1) # single vector per query
Sim = torch.mm(F.normalize(code_set, p=2, dim=1),
F.normalize(code_f, p=2, dim=1).t())
# output = logisticReg(Sim.unsqueeze(-1)).squeeze()
_, accuracy = loss_fn(Sim, len(code_f), m_set)
tot_acc += accuracy
vs.append(code_set)
vf.append(code_f)
tg.append(target)
if (batch_idx + 1) % n_batch_verif == 0:
idx += 1
vs = torch.stack(vs).flatten(start_dim=0, end_dim=1)
vf = torch.stack(vf).flatten(start_dim=0, end_dim=1)
tg = torch.stack(tg).flatten(start_dim=0, end_dim=1)
Ptp01[idx], Ptp05[idx], Ptp1[idx], AUC[
idx] = acc_authentication(model, H0_id_v, H0_data_v, tg,
vf.size(0), vs, vf, m_set,
n_batch_verif)
vs, vf, tg = [], [], []
def __init__(self, n_channel=3, num_class=conf.num_class, image_size=conf.input_size):
self.n_channel = n_channel
self.num_class = num_class
self.image_size = image_size
self.images = tf.placeholder(dtype=tf.float32, shape=[None, self.image_size[0], self.image_size[1], self.n_channel],name='images')
num_bbox = 4
if conf.use_secondbig_loss_constrain:
num_bbox = 8
self.loc_trues = tf.placeholder(dtype=tf.float32, shape=[None, None, num_bbox], name='loc_target') ##############
self.cls_trues = tf.placeholder(dtype=tf.float32, shape=[None, None], name='cls_target') ##############
self.is_training = tf.placeholder(dtype=tf.bool, name='traing_mode')
self.global_step = tf.Variable(0, dtype=tf.int64, name='global_step')
#self.logits = self.model(self.images, is_training=self.is_training)
model = RetinaNet(conf.net)
#self.loc_preds, self.cls_preds = model(self.images, training=self.is_training) #retinanet1
self.loc_preds, self.cls_preds = model(self.images, is_training=self.is_training) #retinanet2
self.anchor_boxes = BoxEncoder().get_anchor_boxes(self.image_size)
self.d_bboxes, self.d_cls_pred, self.d_score = BoxEncoder().decode(self.loc_preds[0], self.cls_preds[0], self.image_size, tf_box_order=conf.tf_box_order)
self.loc_loss, self.cls_loss, self.iou_loss = loss_fn(self.loc_preds, self.loc_trues, self.cls_preds, self.cls_trues, self.anchor_boxes, num_classes=self.num_class)
#self.loc_loss, self.cls_loss = loss_fn(self.loc_preds, self.loc_trues, self.cls_preds, self.cls_trues, num_classes=self.num_class)
self.regularization_loss = tf.losses.get_regularization_loss()
tf.add_to_collection('losses', self.loc_loss)
tf.add_to_collection('losses', self.cls_loss)
tf.add_to_collection('losses', self.iou_loss)
tf.add_to_collection('losses', self.regularization_loss)
self.total_loss = tf.add_n(tf.get_collection('losses'))
tf.summary.scalar('loc_loss', self.loc_loss)
tf.summary.scalar('cls_loss', self.cls_loss)
tf.summary.scalar('total_loss', self.total_loss)
# lr = tf.cond(tf.less(self.global_step, 5000),
# lambda: tf.constant(0.0001),
# lambda: tf.cond(tf.less(self.global_step, 8000),
# lambda: tf.constant(0.00005),
# lambda: tf.cond(tf.less(self.global_step, 12000),
# lambda: tf.constant(0.000025),
# lambda: tf.constant(0.00001))))
#lr = tf.train.exponential_decay(0.0001, self.global_step, 1000, 0.96, staircase=True)
#self.optimizer = tf.train.AdamOptimizer(learning_rate=lr)#.minimize(self.avg_loss, global_step=self.global_step)
#self.optimizer=tf.train.GradientDescentOptimizer(learning_rate=lr)#.minimize(self.avg_loss, global_step=self.global_step)
#self.optimizer = tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9, use_nesterov=True)
self.lr = tf.Variable(float(1e-5), trainable=False, dtype=tf.float32)
self.learning_rate_decay_op = self.lr.assign(self.lr * 0.1)
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
tf.summary.scalar('learning_rate', self.lr)
# update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# with tf.control_dependencies(update_ops):
# #self.train_op = self.optimizer.minimize(self.total_loss, global_step=self.global_step)
#
# grads = self.optimizer.compute_gradients(self.total_loss)
# for i, (g, v) in enumerate(grads):
# if g is not None:
# grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients
# self.train_op = self.optimizer.apply_gradients(grads, global_step=self.global_step)
vars1 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)[180:]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
if args.grad_clip_norm is not None:
grads_and_vars = self.optimizer.compute_gradients(self.total_loss)
grads = [x[0] for x in grads_and_vars]
vars = [x[1] for x in grads_and_vars]
grads, _ = tf.clip_by_global_norm(grads, args.grad_clip_norm)
self.train_op = self.optimizer.apply_gradients(zip(grads, vars), global_step=self.global_step)
else:
self.train_op = self.optimizer.minimize(self.total_loss, global_step=self.global_step)
#self.train_op = self.optimizer.minimize(self.total_loss, var_list=vars1, global_step=self.global_step)
tf.add_to_collection('input_tensor',self.images)
tf.add_to_collection('input_tensor',self.loc_trues)
tf.add_to_collection('input_tensor',self.cls_trues)
tf.add_to_collection('input_tensor',self.is_training)
tf.add_to_collection('output_tensor', self.loc_preds)
tf.add_to_collection('output_tensor', self.cls_preds)
tf.add_to_collection('decode_tensor', self.d_bboxes)
tf.add_to_collection('decode_tensor', self.d_cls_pred)
tf.add_to_collection('decode_tensor', self.d_score)
shuffle=params["train_shuffle"])
validation_loader = DataLoader(validation_dataset,
batch_size=params["train_batch_size"],
shuffle=params["train_shuffle"])
test_loader = DataLoader(test_dataset,
batch_size=params["test_batch_size"],
shuffle=params["test_shuffle"])
if __TIME__:
log_info_time("generate dataloader time \t: ",
datetime.timedelta(seconds=time.time() - start_time))
'''
Setting Loss Function
'''
if __TIME__:
start_time = time.time()
criterion = loss_fn(hyper_params["loss_fn"])
inner_criterion = loss_fn(meta_params["inner_loss"])
outer_criterion = loss_fn(meta_params["outer_loss"])
# if torch.cuda.is_available():
# TODO: implement parallel training
# if options["parallel_criterion"] :
# print(options["parallel_criterion_comment"])
# criterion = DataParallelCriterion(criterion,device_ids=[0, 1, 2])
if __TIME__:
log_info_time("setting loss func time \t: ",
datetime.timedelta(seconds=time.time() - start_time))
'''
Setting Optimizer
'''
if __TIME__:
