def preeval_batch(self, test_loader, abs_len, num_exams):
torch.set_grad_enabled(False)
refs = {}
cands = []
for i in range(num_exams):
cands.append({})
i = 0
for batch_idx, (source, target, input_lengths) in enumerate(test_loader):
for j in range(source.size(0)):
i += 1
ref = self.prepare_for_bleu(target[j])
refs[i] = [ref]
input_variables = source
input_lengths = input_lengths.tolist()
prev_generated_seq = None
for k in range(num_exams):
_, _, other = \
self.model(input_variables, prev_generated_seq, input_lengths)
length = other['length']
sequence = torch.stack(other['sequence'], 1).squeeze(2)
prev_generated_seq = self._mask(sequence)
for j in range(len(length)):
out_seq = [other['sequence'][di][j] for di in range(length[j])]
out = self.prepare_for_bleu(out_seq)
cands[k][i] = out
if i % 100 == 0:
print("Percentages: %.4f" % (i/float(abs_len)))
return cands, refs
def eval(self) -> None:
"""
Sets the model to evaluation/inference mode, disabling dropout and
gradient calculation.
"""
self.nn.eval()
torch.set_grad_enabled(False)
def occlusion_sensitivity(
model, images, ids, mean=None, patch=35, stride=1, n_batches=128
):
"""
"Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization"
https://arxiv.org/pdf/1610.02391.pdf
Look at Figure A5 on page 17
Originally proposed in:
"Visualizing and Understanding Convolutional Networks"
https://arxiv.org/abs/1311.2901
"""
torch.set_grad_enabled(False)
model.eval()
mean = mean if mean else 0
patch_H, patch_W = patch if isinstance(patch, Sequence) else (patch, patch)
pad_H, pad_W = patch_H // 2, patch_W // 2
# Padded image
images = F.pad(images, (pad_W, pad_W, pad_H, pad_H), value=mean)
B, _, H, W = images.shape
new_H = (H - patch_H) // stride + 1
new_W = (W - patch_W) // stride + 1
# Prepare sampling grids
anchors = []
grid_h = 0
while grid_h <= H - patch_H:
grid_w = 0
while grid_w <= W - patch_W:
grid_w += stride
anchors.append((grid_h, grid_w))
grid_h += stride
# Baseline score without occlusion
baseline = model(images).detach().gather(1, ids)
# Compute per-pixel logits
scoremaps = []
for i in tqdm(range(0, len(anchors), n_batches), leave=False):
batch_images = []
batch_ids = []
for grid_h, grid_w in anchors[i : i + n_batches]:
images_ = images.clone()
images_[..., grid_h : grid_h + patch_H, grid_w : grid_w + patch_W] = mean
batch_images.append(images_)
batch_ids.append(ids)
batch_images = torch.cat(batch_images, dim=0)
batch_ids = torch.cat(batch_ids, dim=0)
scores = model(batch_images).detach().gather(1, batch_ids)
scoremaps += list(torch.split(scores, B))
diffmaps = torch.cat(scoremaps, dim=1) - baseline
diffmaps = diffmaps.view(B, new_H, new_W)
return diffmaps
def train(self) -> None:
"""
Sets the model to training mode (enables dropout layers and disables
softmax on CTC layers).
"""
self.nn.train()
# set last layer back to eval mode if not CTC output layer
# (log_softmax/softmax switch).
if not self.criterion:
self.nn[-1].eval()
torch.set_grad_enabled(True)
def _worker(i, module, input, kwargs, device=None):
torch.set_grad_enabled(grad_enabled)
if device is None:
device = get_a_var(input).get_device()
try:
with torch.cuda.device(device):
output = module(*input, **kwargs)
with lock:
results[i] = output
except Exception as e:
with lock:
results[i] = e
def predict_seq_title(self, title, sec_seq, num_exams):
""" Make prediction given `src_seq` as input.
Args:
src_seq (list): list of tokens in source language
Returns:
tgt_seq (list): list of tokens in target language as predicted
by the pre-trained model
"""
torch.set_grad_enabled(False)
text = []
for tok in title:
if tok in self.vectorizer.word2idx:
text.append(self.vectorizer.word2idx[tok])
else:
text.append(3)
input_variable = torch.LongTensor(text).view(1, -1)
if torch.cuda.is_available():
input_variable = input_variable.cuda()
input_lengths = [len(title)]
text = []
for tok in sec_seq:
if tok in self.vectorizer.word2idx:
text.append(self.vectorizer.word2idx[tok])
else:
text.append(3)
prev_generated_seq = torch.LongTensor(text).view(1, -1)
if torch.cuda.is_available():
prev_generated_seq = prev_generated_seq.cuda()
outputs = []
for i in range(num_exams):
_, _, other = \
self.model(input_variable, prev_generated_seq, input_lengths)
length = other['length'][0]
tgt_id_seq = [other['sequence'][di][0].item() for di in range(length)]
tgt_seq = [self.vectorizer.idx2word[tok] for tok in tgt_id_seq]
output = ' '.join([i for i in tgt_seq if i != '<PAD>' and i != '<EOS>' and i != '<SOS>'])
outputs.append(output)
prev_generated_seq = torch.LongTensor(tgt_id_seq).view(1, -1)
if torch.cuda.is_available():
prev_generated_seq = prev_generated_seq.cuda()
return outputs
def _worker(i, module, input, kwargs, device=None):
torch.set_grad_enabled(grad_enabled)
if device is None:
device = get_a_var(input).get_device()
try:
with torch.cuda.device(device):
# this also avoids accidental slicing of `input` if it is a Tensor
if not isinstance(input, (list, tuple)):
input = (input,)
output = module(*input, **kwargs)
with lock:
results[i] = output
except Exception as e:
with lock:
results[i] = e
def _make_images_board(self, model):
model.eval()
num_imgs = 64
fuseTrans = self.cfg.fuseTrans
batch = next(iter(self.data_loaders[1]))
input_images, renderTrans, depthGT, maskGT = utils.unpack_batch_novel(batch, self.cfg.device)
with torch.set_grad_enabled(False):
XYZ, maskLogit = model(input_images)
# ------ build transformer ------
XYZid, ML = transform.fuse3D(
self.cfg, XYZ, maskLogit, fuseTrans) # [B,3,VHW],[B,1,VHW]
newDepth, newMaskLogit, collision = transform.render2D(
self.cfg, XYZid, ML, renderTrans) # [B,N,1,H,W]
return {'RGB': utils.make_grid( input_images[:num_imgs]),
'depth': utils.make_grid(
((1-newDepth)*(collision==1).float())[:num_imgs, 0, 0:1, :, :]),
'depthGT': utils.make_grid(
1-depthGT[:num_imgs, 0, 0:1, :, :]),
'mask': utils.make_grid(
torch.sigmoid(maskLogit[:num_imgs, 0:1,:, :])),
'mask_rendered': utils.make_grid(
torch.sigmoid(newMaskLogit[:num_imgs, 0, 0:1, :, :])),
'maskGT': utils.make_grid(
maskGT[:num_imgs, 0, 0:1, :, :]),
}
def eval_dist(self):
print("======= EVALUATION START =======")
CADN = len(self.CADs)
pred2GT_all = np.ones([CADN, self.cfg.inputViewN]) * np.inf
GT2pred_all = np.ones([CADN, self.cfg.inputViewN]) * np.inf
with torch.set_grad_enabled(False):
for m, cad in enumerate(self.CADs):
# load GT
obj = scipy.io.loadmat(f"{self.cfg.path}/{self.cfg.category}_testGT/{cad}.mat")
Vgt = torch.from_numpy(np.concatenate([obj["V"], obj["Vd"]], axis=0)).to(self.device).float()
VgtN = len(Vgt)
# load prediction
Vpred24 = scipy.io.loadmat(f"{self.result_path}/{cad}.mat")["pointcloud"][:, 0]
assert (len(Vpred24) == self.cfg.inputViewN)
for a in range(self.cfg.inputViewN):
Vpred = torch.from_numpy(Vpred24[a]).to(self.device).float()
VpredN = len(Vpred)
# rotate CAD model to be in consistent coordinates
Vpred[:, 1], Vpred[:, 2] = Vpred[:, 2], -Vpred[:, 1]
# compute test error in both directions
pred2GT_all[m, a] = self._computeTestError(Vpred, Vgt, type="pred->GT")
GT2pred_all[m, a] = self._computeTestError(Vgt, Vpred, type="GT->pred")
info = {"cad": cad,
"pred->GT": pred2GT_all[m].mean()*100,
"GT->pred": GT2pred_all[m].mean()*100,}
print(info)
self.history.append(info)
print("======= EVALUATION DONE =======")
return pd.DataFrame(self.history)
def _get_state_action_values(self, transitions):
batch_size = len(transitions)
batch = Transition(*zip(*transitions))
non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), dtype=torch.uint8, device=self.config.device)
state_batch = torch.cat(batch.state).to(torch.float32)
action_batch = torch.cat(batch.action)
reward_batch = torch.cat(batch.reward).to(torch.float32)
next_state_values = torch.zeros(batch_size).to(self.config.device, dtype=torch.float32)
next_states = [s for s in batch.next_state if s is not None]
if len(next_states) != 0:
with torch.no_grad():
non_final_next_state = torch.cat(next_states).to(torch.float32)
Q = self._get_Q(self.model, non_final_next_state)
best_actions = torch.argmax(Q, dim=1, keepdim=True)
Q_target = self._get_Q(self.target_model, non_final_next_state)
next_state_values[non_final_mask] = Q_target.gather(1, best_actions).squeeze()
gamma = self.config.gamma ** self.config.num_multi_step_reward
expected_values = reward_batch + gamma * next_state_values
with torch.set_grad_enabled(self.model.training):
Q = self._get_Q(self.model, state_batch)
values = torch.squeeze(Q.gather(1, action_batch))
values.to(self.config.device, dtype=torch.float32)
return (values, expected_values)
def train_batch(param):
if len(memory) < param['batch_size']:
return 0
batch = memory.sample(param['batch_size'])
batch_states = default_states_preprocessor([m.state for m in batch])
batch_next_states = default_states_preprocessor([m.next_state for m in batch])
batch_ended = torch.tensor([m.ended for m in batch])
batch_rewards = torch.tensor([m.reward for m in batch]).to(device)
batch_actions = torch.tensor([m.action for m in batch]).to(device)
## Calculate expected reward:
with torch.set_grad_enabled(False):
not_ended_batch = 1 -torch.ByteTensor(batch_ended).to(device)
next_states_non_final = batch_next_states[not_ended_batch]
next_state_values = torch.zeros(param['batch_size']).to(device)
reward_hat = target_dqn(next_states_non_final)
next_state_values[not_ended_batch] = reward_hat.max(1)[0]
expected_state_action_values = next_state_values*param['GAMMA'] + batch_rewards
# Predict value function:
yhat = dqn(batch_states)
state_action_values = yhat.gather(1, batch_actions.unsqueeze(1)).squeeze()
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values)
optimizer.zero_grad()
loss.backward()
for param in dqn.parameters():
param.data.clamp_(-1, 1)
optimizer.step()
return float(loss.data.cpu().numpy())
def test():
test_transform = Compose([PrepareImageAndMask(),
ResizeToNxN(img_size) if args.resize else PadToNxN(img_size), HWCtoCHW()])
test_dataset = SaltIdentification(mode='test', transform=test_transform, preload=False)
test_dataloader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.dataload_workers_nums)
flipped_test_transform = Compose([PrepareImageAndMask(), HorizontalFlip(),
ResizeToNxN(img_size) if args.resize else PadToNxN(img_size), HWCtoCHW()])
flipped_test_dataset = SaltIdentification(mode='test', transform=flipped_test_transform, preload=False)
flipped_test_dataloader_iter = iter(DataLoader(flipped_test_dataset, batch_size=args.batch_size,
num_workers=args.dataload_workers_nums))
model.eval()
torch.set_grad_enabled(False)
prediction = {}
submission = {}
pbar = tqdm(test_dataloader, unit="images", unit_scale=test_dataloader.batch_size, disable=None)
empty_images_count = 0
for batch in pbar:
if args.tta:
flipped_batch = next(flipped_test_dataloader_iter)
else:
flipped_batch = None
probs = predict(model, batch, flipped_batch, use_gpu=use_gpu)
pred = probs > args.threshold
empty_images_count += (pred.sum(axis=(1, 2)) == 0).sum()
probs_uint16 = (65535 * probs).astype(dtype=np.uint16)
image_ids = batch['image_id']
prediction.update(dict(zip(image_ids, probs_uint16)))
rle = rlenc_np(pred)
submission.update(dict(zip(image_ids, rle)))
empty_images_percentage = empty_images_count / len(prediction)
print("empty images: %.2f%% (in public LB 38%%)" % (100 * empty_images_percentage))
gzip_save('-'.join([args.output_prefix, 'probabilities.pkl.gz']), prediction)
sub = pd.DataFrame.from_dict(submission, orient='index')
sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('-'.join([args.output_prefix, 'submission.csv']))
def _train_on_epoch(self, model, optimizer):
model.train()
data_loader = self.data_loaders[0]
running_loss_depth = 0.0
running_loss_mask = 0.0
running_loss = 0.0
fuseTrans = self.cfg.fuseTrans
for self.iteration, batch in enumerate(data_loader, self.iteration):
input_images, renderTrans, depthGT, maskGT = utils.unpack_batch_novel(batch, self.cfg.device)
with torch.set_grad_enabled(True):
optimizer.zero_grad()
XYZ, maskLogit = model(input_images)
# ------ build transformer ------
XYZid, ML = transform.fuse3D(
self.cfg, XYZ, maskLogit, fuseTrans) # [B,3,VHW],[B,1,VHW]
newDepth, newMaskLogit, collision = transform.render2D(
self.cfg, XYZid, ML, renderTrans) # [B,N,H,W,1]
# ------ Compute loss ------
loss_depth = self.l1(
newDepth.masked_select(collision==1),
depthGT.masked_select(collision==1))
loss_mask = self.sigmoid_bce(newMaskLogit, maskGT)
loss = loss_mask + self.cfg.lambdaDepth * loss_depth
# Update weights
loss.backward()
# True Weight decay
if self.cfg.trueWD is not None:
for group in optimizer.param_groups:
for param in group['params']:
param.data = param.data.add(
-self.cfg.trueWD * group['lr'], param.data)
optimizer.step()
if self.on_after_batch is not None:
if self.cfg.lrSched.lower() in "cyclical":
self.on_after_batch(self.iteration)
else: self.on_after_batch(self.epoch)
running_loss_depth += loss_depth.item() * input_images.size(0)
running_loss_mask += loss_mask.item() * input_images.size(0)
running_loss += loss.item() * input_images.size(0)
epoch_loss_depth = running_loss_depth / len(data_loader.dataset)
epoch_loss_mask = running_loss_mask / len(data_loader.dataset)
epoch_loss = running_loss / len(data_loader.dataset)
print(f"\tTrain loss: {epoch_loss}")
return {"epoch_loss_depth": epoch_loss_depth,
"epoch_loss_mask": epoch_loss_mask,
"epoch_loss": epoch_loss, }
def findLR(self, model, optimizer, writer,
start_lr=1e-7, end_lr=10, num_iters=50):
model.train()
losses = []
lrs = np.logspace(np.log10(start_lr), np.log10(end_lr), num_iters)
for lr in lrs:
# Update LR
for group in optimizer.param_groups: group['lr'] = lr
batch = next(iter(self.data_loaders[0]))
input_images, depthGT, maskGT = utils.unpack_batch_fixed(batch, self.cfg.device)
# ------ define ground truth------
XGT, YGT = torch.meshgrid([torch.arange(self.cfg.outH), # [H,W]
torch.arange(self.cfg.outW)]) # [H,W]
XGT, YGT = XGT.float(), YGT.float()
XYGT = torch.cat([
XGT.repeat([self.cfg.outViewN, 1, 1]),
YGT.repeat([self.cfg.outViewN, 1, 1])], dim=0) #[2V,H,W]
XYGT = XYGT.unsqueeze(dim=0).to(self.cfg.device) #[1,2V,H,W]
with torch.set_grad_enabled(True):
optimizer.zero_grad()
XYZ, maskLogit = model(input_images)
XY = XYZ[:, :self.cfg.outViewN * 2, :, :]
depth = XYZ[:, self.cfg.outViewN * 2:self.cfg.outViewN * 3, :, :]
mask = (maskLogit > 0).byte()
# ------ Compute loss ------
loss_XYZ = self.l1(XY, XYGT)
loss_XYZ += self.l1(depth.masked_select(mask),
depthGT.masked_select(mask))
loss_mask = self.sigmoid_bce(maskLogit, maskGT)
loss = loss_mask + self.cfg.lambdaDepth * loss_XYZ
# Update weights
loss.backward()
# True Weight decay
if self.cfg.trueWD is not None:
for group in optimizer.param_groups:
for param in group['params']:
param.data = param.data.add(
-self.cfg.trueWD * group['lr'], param.data)
optimizer.step()
losses.append(loss.item())
fig, ax = plt.subplots()
ax.plot(lrs, losses)
ax.set_xlabel('learning rate')
ax.set_ylabel('loss')
ax.set_xscale('log')
writer.add_figure('findLR', fig)
def findLR(self, model, optimizer, writer,
start_lr=1e-7, end_lr=10, num_iters=50):
model.train()
lrs = np.logspace(np.log10(start_lr), np.log10(end_lr), num_iters)
losses = []
fuseTrans = self.cfg.fuseTrans
for lr in lrs:
# Update LR
for group in optimizer.param_groups:
group['lr'] = lr
batch = next(iter(self.data_loaders[0]))
input_images, renderTrans, depthGT, maskGT = utils.unpack_batch_novel(batch, self.cfg.device)
with torch.set_grad_enabled(True):
optimizer.zero_grad()
XYZ, maskLogit = model(input_images)
# ------ build transformer ------
XYZid, ML = transform.fuse3D(
self.cfg, XYZ, maskLogit, fuseTrans) # [B,3,VHW],[B,1,VHW]
newDepth, newMaskLogit, collision = transform.render2D(
self.cfg, XYZid, ML, renderTrans) # [B,N,H,W,1]
# ------ Compute loss ------
loss_depth = self.l1(
newDepth.masked_select(collision==1),
depthGT.masked_select(collision==1))
loss_mask = self.sigmoid_bce(newMaskLogit, maskGT)
loss = loss_mask + self.cfg.lambdaDepth * loss_depth
# Update weights
loss.backward()
# True Weight decay
if self.cfg.trueWD is not None:
for group in optimizer.param_groups:
for param in group['params']:
param.data = param.data.add(
-self.cfg.trueWD * group['lr'],
param.data)
optimizer.step()
losses.append(loss.item())
fig, ax = plt.subplots()
ax.plot(lrs, losses)
ax.set_xlabel('learning rate')
ax.set_ylabel('loss')
ax.set_xscale('log')
writer.add_figure('findLR', fig)
def predict_batch(self, source, input_lengths, num_exams):
torch.set_grad_enabled(False)
output_seq = []
input_variables = source
for i in range(source.size(0)):
title_id_seq = [input_variables[i][di].item() for di in range(input_lengths[i])]
title_seq = [self.vectorizer.idx2word[tok] for tok in title_id_seq]
title = ' '.join([k for k in title_seq if k != '<PAD>' and k != '<EOS>' and k != '<SOS>'])
output_seq.append([title])
prev_generated_seq = None
for k in range(num_exams):
_, _, other = \
self.model(input_variables, prev_generated_seq, input_lengths)
length = other['length']
sequence = torch.stack(other['sequence'], 1).squeeze(2)
prev_generated_seq = self._mask(sequence)
for i in range(len(length)):
opt_id_seq = [other['sequence'][di][i].item() for di in range(length[i])]
opt_seq = [self.vectorizer.idx2word[tok] for tok in opt_id_seq]
output = ' '.join([k for k in opt_seq if k != '<PAD>' and k != '<EOS>' and k != '<SOS>'])
output_seq[i].append(output)
return output_seq
def contractive_penalty(self, network, input, penalty_amount=0.5):
if penalty_amount == 0.:
return
if not isinstance(input, (list, tuple)):
input = [input]
input = [inp.detach() for inp in input]
input = [inp.requires_grad_() for inp in input]
with torch.set_grad_enabled(True):
output = network(*input)
gradient = self._get_gradient(input, output)
gradient = gradient.view(gradient.size()[0], -1)
penalty = (gradient ** 2).sum(1).mean()
return penalty_amount * penalty
def _val_on_epoch(self, model):
model.eval()
data_loader = self.data_loaders[1]
running_loss_XYZ = 0.0
running_loss_mask = 0.0
running_loss = 0.0
for batch in data_loader:
input_images, depthGT, maskGT = utils.unpack_batch_fixed(batch, self.cfg.device)
# ------ define ground truth------
XGT, YGT = torch.meshgrid([
torch.arange(self.cfg.outH), # [H,W]
torch.arange(self.cfg.outW)]) # [H,W]
XGT, YGT = XGT.float(), YGT.float()
XYGT = torch.cat([
XGT.repeat([self.cfg.outViewN, 1, 1]),
YGT.repeat([self.cfg.outViewN, 1, 1])], dim=0) #[2V,H,W]
XYGT = XYGT.unsqueeze(dim=0).to(self.cfg.device) # [1,2V,H,W]
with torch.set_grad_enabled(False):
XYZ, maskLogit = model(input_images)
XY = XYZ[:, :self.cfg.outViewN * 2, :, :]
depth = XYZ[:, self.cfg.outViewN * 2:self.cfg.outViewN*3,:,:]
mask = (maskLogit > 0).byte()
# ------ Compute loss ------
loss_XYZ = self.l1(XY, XYGT)
loss_XYZ += self.l1(depth.masked_select(mask),
depthGT.masked_select(mask))
loss_mask = self.sigmoid_bce(maskLogit, maskGT)
loss = loss_mask + self.cfg.lambdaDepth * loss_XYZ
running_loss_XYZ += loss_XYZ.item() * input_images.size(0)
running_loss_mask += loss_mask.item() * input_images.size(0)
running_loss += loss.item() * input_images.size(0)
epoch_loss_XYZ = running_loss_XYZ / len(data_loader.dataset)
epoch_loss_mask = running_loss_mask / len(data_loader.dataset)
epoch_loss = running_loss / len(data_loader.dataset)
print(f"\tVal loss: {epoch_loss}")
return {"epoch_loss_XYZ": epoch_loss_XYZ,
"epoch_loss_mask": epoch_loss_mask,
"epoch_loss": epoch_loss, }
def _val_on_epoch(self, model):
model.eval()
data_loader = self.data_loaders[1]
running_loss_depth = 0.0
running_loss_mask = 0.0
running_loss = 0.0
fuseTrans = self.cfg.fuseTrans
for batch in data_loader:
input_images, renderTrans, depthGT, maskGT = utils.unpack_batch_novel(batch, self.cfg.device)
with torch.set_grad_enabled(False):
XYZ, maskLogit = model(input_images)
# ------ build transformer ------
XYZid, ML = transform.fuse3D(
self.cfg, XYZ, maskLogit, fuseTrans) # [B,3,VHW],[B,1,VHW]
newDepth, newMaskLogit, collision = transform.render2D(
self.cfg, XYZid, ML, renderTrans) # [B,N,H,W,1]
# ------ Compute loss ------
loss_depth = self.l1(
newDepth.masked_select(collision==1),
depthGT.masked_select(collision==1))
loss_mask = self.sigmoid_bce(newMaskLogit, maskGT)
loss = loss_mask + self.cfg.lambdaDepth * loss_depth
running_loss_depth += loss_depth.item() * input_images.size(0)
running_loss_mask += loss_mask.item() * input_images.size(0)
running_loss += loss.item() * input_images.size(0)
epoch_loss_depth = running_loss_depth / len(data_loader.dataset)
epoch_loss_mask = running_loss_mask / len(data_loader.dataset)
epoch_loss = running_loss / len(data_loader.dataset)
print(f"\tVal loss: {epoch_loss}")
return {"epoch_loss_depth": epoch_loss_depth,
"epoch_loss_mask": epoch_loss_mask,
"epoch_loss": epoch_loss, }
def _make_images_board(self, model):
model.eval()
num_imgs = 64
batch = next(iter(self.data_loaders[1]))
input_images, depthGT, maskGT = utils.unpack_batch_fixed(batch, self.cfg.device)
with torch.set_grad_enabled(False):
XYZ, maskLogit = model(input_images)
XY = XYZ[:, :self.cfg.outViewN * 2, :, :]
depth = XYZ[:, self.cfg.outViewN * 2:self.cfg.outViewN * 3, :, :]
mask = (maskLogit > 0).float()
return {'RGB': utils.make_grid(input_images[:num_imgs]),
'depth': utils.make_grid(1-depth[:num_imgs, 0:1, :, :]),
'depth_mask': utils.make_grid(
((1-depth)*mask)[:num_imgs, 0:1, :, :]),
'depthGT': utils.make_grid(
1-depthGT[:num_imgs, 0:1, :, :]),
'mask': utils.make_grid(
torch.sigmoid(maskLogit[:num_imgs, 0:1,:, :])),
'maskGT': utils.make_grid(maskGT[:num_imgs, 0:1, :, :]),
}
def interpolate_penalty(self, network, input, penalty_amount=0.5):
input = input.detach()
input = input.requires_grad_()
if len(input) != 2:
raise ValueError('tuple of 2 inputs required to interpolate')
inp1, inp2 = input
try:
epsilon = network.inputs.e.view(-1, 1, 1, 1)
except AttributeError:
raise ValueError('You must initiate a uniform random variable'
'`e` to use interpolation')
mid_in = ((1. - epsilon) * inp1 + epsilon * inp2)
mid_in.requires_grad_()
with torch.set_grad_enabled(True):
mid_out = network(mid_in)
gradient = self._get_gradient(mid_in, mid_out)
gradient = gradient.view(gradient.size()[0], -1)
penalty = ((gradient.norm(2, dim=1) - 1.) ** 2).mean()
return penalty_amount * penalty
def step(self, data, mode='train'):
torch.cuda.empty_cache()
if self.speed_benchmark:
step_start = time.time()
with torch.set_grad_enabled(mode == 'train'):
np.random.seed()
self.optimizerF.zero_grad()
MSEcriterion = torch.nn.MSELoss()
BCEcriterion = torch.nn.BCELoss()
CEcriterion = nn.CrossEntropyLoss(
weight=self.class_balance_weights, reduce=False)
rgb, norm, depth, dataMask, Q = v(data['rgb']), v(data['norm']), v(
data['depth']), v(data['dataMask']), v(data['Q'])
segm = v(data['segm'])
segm = torch.cat((segm[:, 0, :, :, :], segm[:, 1, :, :, :]))
errG_rgb, errG_d, errG_n, errG_k, errG_s = torch.FloatTensor(
[0]), torch.FloatTensor([0]), torch.FloatTensor(
[0]), torch.FloatTensor([0]), torch.FloatTensor([0])
n = Q.shape[0]
# compose the input: [rgb, normal, depth]
complete0 = torch.cat(
(rgb[:, 0, :, :, :], norm[:, 0, :, :, :], depth[:, 0:1, :, :]),
1)
complete1 = torch.cat(
(rgb[:, 1, :, :, :], norm[:, 1, :, :, :], depth[:, 1:2, :, :]),
1)
view0, mask0, geow0 = apply_mask(complete0.clone(),
self.args.maskMethod,
self.args.ObserveRatio)
view1, mask1, geow1 = apply_mask(complete1.clone(),
self.args.maskMethod,
self.args.ObserveRatio)
view = torch.cat((view0, view1))
mask = torch.cat((mask0, mask1))
# mask the pano
complete = torch.cat((complete0, complete1))
dataMask = torch.cat(
(dataMask[:, 0, :, :, :], dataMask[:, 1, :, :, :]))
fakec = self.netF(complete)
segm_pred = self.netSemg(fakec)
featMapsc = fakec[:n]
featMaptc = fakec[n:]
denseCorres = data['denseCorres']
validCorres = torch.nonzero(
denseCorres['valid'] == 1).view(-1).long()
if not len(validCorres):
loss_fl_pos = torch_op.v(np.array([0]))[0]
loss_fl_neg = torch_op.v(np.array([0]))[0]
loss_fl = torch_op.v(np.array([0]))[0][0]
loss_fc = torch_op.v(np.array([0]))[0]
loss_fl_pos_f = torch_op.v(np.array([0]))[0]
loss_fl_neg_f = torch_op.v(np.array([0]))[0]
loss_fl_f = torch_op.v(np.array([0]))[0]
else:
# categorize each correspondence by whether it contain unobserved point
allCorres = torch.cat(
(denseCorres['idxSrc'], denseCorres['idxTgt']))
corresShape = allCorres.shape
allCorres = allCorres.view(-1, 2).long()
typeIdx = torch.arange(corresShape[0]).view(-1, 1).repeat(
1, corresShape[1]).view(-1).long()
typeIcorresP = mask[typeIdx, 0, allCorres[:, 1], allCorres[:,
0]]
typeIcorresP = typeIcorresP.view(2, -1, corresShape[1]).sum(0)
denseCorres['observe'] = typeIcorresP
# consistency of keypoint proposal across different view
idxInst = torch.arange(n)[validCorres].view(-1, 1).repeat(
1, denseCorres['idxSrc'].shape[1]).view(-1).long()
featS = featMapsc[idxInst, :,
denseCorres['idxSrc'][validCorres, :,
1].view(-1).long(),
denseCorres['idxSrc'][validCorres, :,
0].view(-1).long()]
featT = featMaptc[idxInst, :,
denseCorres['idxTgt'][validCorres, :,
1].view(-1).long(),
denseCorres['idxTgt'][validCorres, :,
0].view(-1).long()]
# positive example,
loss_fl_pos = (featS - featT).pow(2).sum(1).mean()
# negative example, make sure does not contain positive
Kn = denseCorres['idxSrc'].shape[1]
C = featMapsc.shape[1]
negIdy = torch.from_numpy(
np.random.choice(range(featMapsc.shape[2]),
Kn * 100 * len(validCorres)))
negIdx = torch.from_numpy(
np.random.choice(range(featMapsc.shape[3]),
Kn * 100 * len(validCorres)))
idx = torch.arange(n)[validCorres].view(-1, 1).repeat(
1, Kn * 100).view(-1).long()
loss_fl_neg = F.relu(self.args.D - (
featS.unsqueeze(1).repeat(1, 100, 1).view(-1, C) -
featMaptc[idx, :, negIdy, negIdx]).pow(2).sum(1)).mean()
loss_fl = loss_fl_pos + loss_fl_neg
errG = loss_fl
total_weight = dataMask
if self.args.featlearnSegm:
errG_s = (CEcriterion(segm_pred,
segm.squeeze(1).long()) *
total_weight).mean() * 0.1
errG += errG_s
if mode == 'train' and len(validCorres) > 0:
errG.backward()
self.optimizerF.step()
self.logger_errG.update(errG.data, Q.size(0))
self.logger_errG_fl.update(loss_fl.data, Q.size(0))
self.logger_errG_fl_pos.update(loss_fl_pos.data, Q.size(0))
self.logger_errG_fl_neg.update(loss_fl_neg.data, Q.size(0))
suffix = f"| errG {self.logger_errG.avg:.6f}| errG_fl {self.logger_errG_fl.avg:.6f}\
| errG_fl_pos {self.logger_errG_fl_pos.avg:.6f} | errG_fl_neg {self.logger_errG_fl_neg.avg:.6f} | errG_fc {self.logger_errG_fc.avg:.6f} | errG_pn {self.logger_errG_pn.avg:.6f} | errG_freq {self.logger_errG_freq.avg:.6f}"
if self.global_step % getattr(self.learnerParam,
f"{mode}_step_vis") == 0:
if mode != 'train':
with torch.set_grad_enabled(False):
if len(validCorres):
self.evalFeatRatioSift.extend(
self.evalSiftDescriptor(rgb, denseCorres))
obs, unobs, visCPc = self.evalDLDescriptor(
featMapsc, featMaptc, denseCorres,
complete0[:, 0:3, :, :],
complete1[:, 0:3, :, :], mask[0:1, 0:1, :, :])
self.evalFeatRatioDLc_obs.extend(obs)
self.evalFeatRatioDLc_unobs.extend(unobs)
visCPdir = os.path.join(
self.args.EXP_DIR_SAMPLES,
f"step_{self.global_step}")
if not os.path.exists(visCPdir): os.mkdir(visCPdir)
for ii in range(len(visCPc)):
cv2.imwrite(
os.path.join(visCPdir,
f"complete_{ii}.png"),
visCPc[ii])
vis = []
# draw semantic
viss = segm
vissf = segm_pred
vissf = torch.argmax(vissf, 1, keepdim=True).float()
viss = torch.cat((vissf, viss), 2)
visstp = torch_op.npy(viss)
visstp = np.expand_dims(np.squeeze(visstp, 1), 3)
visstp = self.colors[
visstp.flatten().astype('int'), :].reshape(
visstp.shape[0], visstp.shape[1], visstp.shape[2], 3)
viss = torch_op.v(visstp.transpose(0, 3, 1, 2)) / 255.
vis.append(viss)
vis = torch.cat(vis, 2)[::2]
permute = [2, 1, 0] # bgr to rgb
vis = vis[:, permute, :, :]
train_op.tboard_add_img(self.tensorboardX, vis, f"{mode}/loss",
self.global_step)
if self.global_step % getattr(self.learnerParam,
f"{mode}_step_log") == 0:
self.tensorboardX.add_scalars('data/errG_fl',
{f"{mode}_complete": loss_fl},
self.global_step)
self.tensorboardX.add_scalars(
'data/errG_fl_pos', {f"{mode}_complete": loss_fl_pos},
self.global_step)
self.tensorboardX.add_scalars(
'data/errG_fl_neg', {f"{mode}_complete": loss_fl_neg},
self.global_step)
self.tensorboardX.add_scalars('data/errG_s',
{f"{mode}": errG_s},
self.global_step)
summary = {'suffix': suffix}
self.global_step += 1
if self.speed_benchmark:
self.time_per_step.update(time.time() - step_start, 1)
print(f"time elapse per step: {self.time_per_step.avg}")
return dotdict(summary)
def loop(mode,
data,
outer_steps,
inner_steps,
log_steps,
fig_epochs,
inner_lr,
log_mask=True,
unroll_steps=None,
meta_batchsize=0,
sampler=None,
epoch=1,
outer_optim=None,
save_path=None,
is_RL=False):
#####################################################################
"""Args:
meta_batchsize(int): If meta_batchsize |m| > 0, gradients for multiple
unrollings from each episodes size of |m| will be accumulated in
sequence but updated all at once. (Which can be done in parallel when
VRAM is large enough, but will be simulated in this code.)
If meta_batchsize |m| = 0(default), then update will be performed after
each unrollings.
"""
assert mode in ['train', 'valid', 'test']
assert meta_batchsize >= 0
print(f'Start_of_{mode}.')
if mode == 'train' and unroll_steps is None:
raise Exception("unroll_steps has to be specied when mode='train'.")
if mode != 'train' and unroll_steps is not None:
raise Warning("unroll_steps has no effect when mode mode!='train'.")
train = True if mode == 'train' else False
force_base = True
# TODO: to gin configuration
fc_pulling = False # does not support for now
class_balanced = False
if class_balanced:
loader_cfg = LoaderConfig(class_size=10, sample_size=3, num_workers=4)
else:
loader_cfg = LoaderConfig(batch_size=128, num_workers=4)
sample_split_ratio = 0.5
# sample_split_ratio = None
anneal_outer_steps = 50
concrete_resample = True
detach_param = False
split_method = {1: 'inclusive', 2: 'exclusive'}[2]
sampler_type = {1: 'pre_sampler', 2: 'post_sampler'}[2]
#####################################################################
mask_unit = {
0: MaskUnit.SAMPLE,
1: MaskUnit.CLASS,
}[0]
mask_dist = {
0: MaskDist.SOFT,
1: MaskDist.DISCRETE,
2: MaskDist.CONCRETE,
3: MaskDist.RL,
}[3 if is_RL else 2]
mask_mode = MaskMode(mask_unit, mask_dist)
#####################################################################
# scheduler
inner_step_scheduler = InnerStepScheduler(outer_steps, inner_steps,
anneal_outer_steps)
if split_method == 'exclusive':
meta_support, meta_query = data.split_classes(
(('Support', 1), ('Query', 5))) # 1(100) : 4(400)
elif split_method == 'inclusive':
meta_support, meta_query = data.split_instances(
(('Support', 5), ('Query', 5))) # 5:5 instances
meta_s_loader0 = meta_support.dataset_loader(loader_cfg)
meta_s_loader1 = meta_support.episode_loader(loader_cfg)
l0 = meta_s_loader0()
print(l0[1])
print(l0[:2])
print(l0.classwise[1])
print(l0.classwise[:2])
l1 = meta_s_loader1()
print(l1[1])
print(l1[:2])
print(l1.classwise[1])
print(l1.classwise[:2])
fake_mask = torch.tensor([True, False] * 5)
l0 = meta_s_loader0()
print(l0)
sample_l0 = l0.masked_select(fake_mask)
print(sample_l0)
class_l0 = l0.classwise.masked_select(fake_mask)
print(class_l0)
l1 = meta_s_loader1()
print(l1)
sample_l1 = l1.masked_select(fake_mask)
print(sample_l1)
class_l1 = l1.classwise.masked_select(fake_mask)
print(class_l1)
# sss.get_classes[:3]
import pdb
pdb.set_trace()
if train:
if meta_batchsize > 0:
# serial processing of meta-minibatch
update_epochs = meta_batchsize
update_steps = None
else:
# update at every unrollings
update_steps = unroll_steps
update_epochs = None
assert (update_epochs is None) != (update_steps is None)
# for result recordin
result_frame = ResultFrame()
if save_path:
writer = SummaryWriter(os.path.join(save_path, 'tfevent'))
##################################
if is_RL:
env = Environment()
policy = Policy(sampler)
neg_rw = None
##################################
for i in range(1, outer_steps + 1):
outer_loss = 0
result_dict = ResultDict()
# initialize sampler
sampler.initialize()
# initialize base learner
model_q = Model(len(meta_support), mode='metric')
if fc_pulling:
model_s = Model(len(meta_support), mode='fc')
params = C(model_s.get_init_params('ours'))
else:
model_s = model_q
params = C(model_s.get_init_params('ours'))
##################################
if is_RL:
policy.reset()
# env.reset()
##################################
if not train or force_base:
"""baseline 0: naive single task learning
baseline 1: single task learning with the same loss scale
"""
# baseline parameters
params_b0 = C(params.copy('b0'), 4)
params_b1 = C(params.copy('b1'), 4)
do_sample = True
for k in range(1, inner_step_scheduler(i, verbose=True)):
outs = []
#####################################################################
if do_sample:
do_sample = False
meta_s_loader = meta_s.get_dataset_loader(
loader_cfg, 'Support')
# task encoding (very first step and right after a meta-update)
with torch.set_grad_enabled(train):
def feature_fn():
# determein whether to untilize model features
if sampler_type == 'post_sampler':
return model_s(meta_s, params)
elif sampler_type == 'pre_sampler':
return None
# sampler do the work!
mask, lr_ = sampler(mask_mode, feature_fn)
# out_for_sampler = model_s(meta_s, params, debug=do_sample)
# mask_, lr = sampler(
# pairwise_dist=out_for_sampler.pairwise_dist,
# classwise_loss=out_for_sampler.loss,
# classwise_acc=out_for_sampler.acc,
# n_classes=out_for_sampler.n_classes,
# mask_mode=mask_mode,
# )
# sample from concrete distribution
# while keeping original distribution for simple resampling
if mask_mode.dist is MaskDist.CONCRETE:
mask = mask_.rsample()
else:
mask = mask_
if is_RL:
mask, neg_rw = policy.predict(mask)
if neg_rw:
# TODO fix iteratively [0 class]s are sampled.
do_sample = True
print('[!] select 0 class!')
policy.rewards.append(neg_rw)
import pdb
pdb.set_trace()
# policy.update()
continue
else:
# we can simply resample masks when using concrete distribution
# without seriously going through the sampler
if mask_mode.dist is MaskDist.CONCRETE and concrete_resample:
mask = mask_.rsample()
#####################################################################
# use learned learning rate if available
# lr = inner_lr if lr is None else lr # inner_lr: preset / lr: learned
# train on support set
params, mask = C([params, mask], 2)
#####################################################################
out_s = model_s(meta_s, params, mask=mask, mask_mode=mask_mode)
# out_s_loss_masked = out_s.loss_masked
outs.append(out_s)
# inner gradient step
out_s_loss_mean = out_s.loss.mean() if mask_mode == MaskMode.RL \
else out_s.loss_masked_mean
out_s_loss_mean, lr = C([out_s_loss_mean, lr], 2)
params = params.sgd_step(
out_s_loss_mean,
lr,
second_order=False if is_RL else True,
detach_param=True if is_RL else detach_param)
#####################################################################
# baseline
if not train or force_base:
out_s_b0 = model_s(meta_s, params_b0)
out_s_b1 = model_s(meta_s, params_b1)
outs.extend([out_s_b0, out_s_b1])
# attach mask to get loss_s
out_s_b1.attach_mask(mask)
# inner gradient step (baseline)
params_b0 = params_b0.sgd_step(out_s_b0.loss.mean(), inner_lr)
params_b1 = params_b1.sgd_step(out_s_b1.loss_scaled_mean,
lr.detach())
meta_s = meta_s.cpu()
meta_q = meta_q.cuda()
# test on query set
with torch.set_grad_enabled(train):
params = C(params, 3)
out_q = model_q(meta_q, params)
outs.append(out_q)
# baseline
if not train or force_base:
with torch.no_grad():
# test on query set
out_q_b0 = model_q(meta_q, params_b0)
out_q_b1 = model_q(meta_q, params_b1)
outs.extend([out_q_b0, out_q_b1])
##################################
if is_RL:
reward = env.step(outs)
policy.rewards.append(reward)
outs.append(policy)
##################################
# record result
result_dict.append(outer_step=epoch * i,
inner_step=k,
**ModelOutput.as_merged_dict(outs))
# append to the dataframe
result_frame = result_frame.append_dict(
result_dict.index_all(-1).mean_all(-1))
# logging
if k % log_steps == 0:
logger.step_info(epoch, mode, i, outer_steps, k,
inner_step_scheduler(i), lr)
# logger.split_info(meta_support, meta_query, episode_iterator)
logger.colorized_mask(mask,
fmt="2d",
vis_num=20,
cond=not sig_1.is_active() and log_mask)
logger.outputs(outs, print_conf=sig_1.is_active())
logger.flush()
# to debug in the middle of running process.
if k == inner_steps and sig_2.is_active():
import pdb
pdb.set_trace()
# compute outer gradient
if train and (k % unroll_steps == 0 or k == inner_steps):
#####################################################################
if not is_RL:
outer_loss += out_q.loss.mean()
outer_loss.backward()
outer_loss = 0
params.detach_().requires_grad_()
mask = mask.detach()
lr = lr.detach()
do_sample = True
#####################################################################
if not train:
if not is_RL:
# when params is not leaf node created by user,
# requires_grad attribute is False by default.
params.requires_grad_()
# meta(outer) learning
if train and update_steps and k % update_steps == 0:
# when you have meta_batchsize == 0, update_steps == unroll_steps
if is_RL:
policy.update()
else:
outer_optim.step()
sampler.zero_grad()
### end of inner steps (k) ###
if train and update_epochs and i % update_epochs == 0:
# when you have meta_batchsize > 0, update_epochs == meta_batchsize
if is_RL:
policy.update()
else:
outer_optim.step()
sampler.zero_grad()
if update_epochs:
print(f'Meta-batchsize is {meta_batchsize}: Sampler updated.')
if train and update_steps:
print(f'Meta-batchsize is zero. Updating after every unrollings.')
# tensorboard
if save_path and train:
step = (epoch * (outer_steps - 1)) + i
res = ResultFrame(result_frame[result_frame['outer_step'] == i])
loss = res.get_best_loss().mean()
acc = res.get_best_acc().mean()
writer.add_scalars('Loss/train', {n: loss[n]
for n in loss.index}, step)
writer.add_scalars('Acc/train', {n: acc[n]
for n in acc.index}, step)
# dump figures
if save_path and i % fig_epochs == 0:
# meta_s.s.save_fig(f'imgs/meta_support', save_path, i)
# meta_q.s.save_fig(f'imgs/meta_query', save_path, i)
result_dict['ours_s_mask'].save_fig(f'imgs/masks', save_path, i)
result_dict.get_items([
'ours_s_mask', 'ours_s_loss', 'ours_s_loss_masked',
'b0_s_loss', 'b1_s_loss', 'ours_q_loss', 'b0_q_loss',
'b1_q_loss'
]).save_csv(f'classwise/{mode}', save_path, i)
# distinguishable episodes
if not i == outer_steps:
print(f'Path for saving: {save_path}')
print(f'End_of_episode: {i}')
### end of episode (i) ###
print(f'End_of_{mode}.')
# del metadata
return sampler, result_frame
def train_model(model, criterion, optimizer, scheduler, num_epochs=no_epochs):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
#each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train() #set model to training mode
else:
model.eval() #set model to evaluate mode
running_loss = 0.0
running_corrects = 0
#iterate over data.
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
#zero the parameter gradients
optimizer.zero_grad()
#forward
#track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
#backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
#statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
epoch_acc))
#deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
#load best model weights
model.load_state_dict(best_model_wts)
return model
return results
def transcribe(audio_path, parser, model, decoder, cuda=False):
spect = parser.parse_audio(audio_path).contiguous()
spect = spect.view(1, 1, spect.size(0), spect.size(1))
if cuda:
spect = spect.cuda()
input_sizes = torch.IntTensor([spect.size(3)]).int()
out, output_sizes = model(spect, input_sizes)
decoded_output, decoded_offsets = decoder.decode(out, output_sizes)
return decoded_output, decoded_offsets
if __name__ == '__main__':
torch.set_grad_enabled(False)
model = DeepSpeech.load_model(args.model_path)
if args.cuda:
model.cuda()
model.eval()
labels = DeepSpeech.get_labels(model)
audio_conf = DeepSpeech.get_audio_conf(model)
if args.decoder == "beam":
from decoder import BeamCTCDecoder
decoder = BeamCTCDecoder(labels, lm_path=args.lm_path, alpha=args.alpha, beta=args.beta,
cutoff_top_n=args.cutoff_top_n, cutoff_prob=args.cutoff_prob,
beam_width=args.beam_width, num_processes=args.lm_workers)
else:
def train(epoch, phase='train'):
global global_step, best_loss, best_metric, best_accuracy
if phase == 'train':
writer.add_scalar('%s/learning_rate' % phase, get_lr(), epoch)
model.train() if phase == 'train' else model.eval()
torch.set_grad_enabled(True) if phase == 'train' else torch.set_grad_enabled(False)
dataloader = train_dataloader if phase == 'train' else valid_dataloader
running_loss, running_metric, running_accuracy = 0.0, 0.0, 0.0
worst_loss, worst_metric = best_loss, best_metric
it, total = 0, 0
if phase == 'valid':
total_probs = []
total_truth = []
pbar_disable = False if epoch == start_epoch else None
pbar = tqdm(dataloader, unit="images", unit_scale=dataloader.batch_size, disable=pbar_disable)
for batch in pbar:
image_ids, inputs, targets = batch['image_id'], batch['input'], batch['mask']
if use_gpu:
inputs = inputs.cuda()
targets = targets.cuda()
# forward
logit, logit_pixel, logit_image = model(inputs)
# look at the center only
if args.loss_on_center:
logit = remove_padding(logit)
logit_pixel = (remove_padding(l) for l in logit_pixel)
targets = remove_padding(targets)
truth_pixel = targets
truth_image = (truth_pixel.sum(dim=(1, 2)) > 0).float()
loss = models.deep_supervised_criterion(logit, logit_pixel, logit_image, truth_pixel, truth_image)
if not args.loss_on_center and not args.resize:
logit = remove_padding(logit)
targets = remove_padding(targets)
probs = torch.sigmoid(logit).squeeze(1)
# predictions = probs.squeeze(1) > 0.5
if phase == 'train':
# backward
optimizer.backward(loss / args.gradient_accumulation)
if it % args.gradient_accumulation == 0:
optimizer.step()
optimizer.zero_grad()
# statistics
it += 1
global_step += 1
loss = loss.item()
running_loss += (loss * targets.size(0))
total += targets.size(0)
writer.add_scalar('%s/loss' % phase, loss, global_step)
targets_numpy = targets.cpu().numpy()
probs_numpy = probs.cpu().detach().numpy()
predictions_numpy = probs_numpy > 0.5 # predictions.cpu().numpy()
metric_array = calc_metric(targets_numpy, predictions_numpy, type='iou', size_average=False)
metric = metric_array.mean()
running_metric += metric_array.sum()
running_accuracy += calc_metric(targets_numpy, predictions_numpy, type='pixel_accuracy',
size_average=False).sum()
if phase == 'valid':
total_truth.append(targets_numpy)
total_probs.append(probs_numpy)
visualize_output = False
if worst_loss > loss:
worst_loss = loss
visualize_output = True
if worst_metric < metric:
worst_metric = metric
visualize_output = True
if visualize_output and args.debug:
# sort samples by metric
ind = np.argsort(metric_array)
images = remove_padding(inputs.cpu())
images = images[ind]
probs = probs[ind].cpu()
predictions = predictions[ind].cpu()
targets = targets[ind].cpu()
preds = torch.cat([probs] * 3, 1)
mask = torch.cat([targets.unsqueeze(1)] * 3, 1)
all = images.clone()
all[:, 0] = torch.max(images[:, 0], predictions.float())
all[:, 1] = torch.max(images[:, 1], targets)
all = torch.cat((torch.cat((all, images), 3), torch.cat((preds, mask), 3)), 2)
all_grid = vutils.make_grid(all, nrow=4, normalize=False, pad_value=1)
writer.add_image('%s/img-mask-pred' % phase, all_grid, global_step)
# update the progress bar
pbar.set_postfix({
'loss': "%.05f" % (running_loss / total),
'metric': "%.03f" % (running_metric / total)
})
epoch_loss = running_loss / total
epoch_metric = running_metric / total
epoch_accuracy = running_accuracy / total
writer.add_scalar('%s/metric' % phase, epoch_metric, epoch)
writer.add_scalar('%s/accuracy' % phase, epoch_accuracy, epoch)
writer.add_scalar('%s/epoch_loss' % phase, epoch_loss, epoch)
if phase == 'valid':
def save_checkpoint(name):
cycle = ('-cycle%d' % (epoch // args.lr_scheduler_step_size)) if args.lr_scheduler == 'clr' else ''
model_name = name + '-model'
model_file_name = '%d-%s-%s%s.pth' % (start_timestamp, model_name, full_name, cycle)
model_file = models_dir / model_file_name
models.save(model, model_file)
mode_file_simple = Path(models_dir / (model_name + '-%s%s.pth' % (args.data_fold, cycle)))
if mode_file_simple.is_symlink() or mode_file_simple.exists():
mode_file_simple.unlink()
mode_file_simple.symlink_to(model_file.relative_to(mode_file_simple.parent))
checkpoint = {
'epoch': epoch,
'step': global_step,
'model_file': str(model_file),
'best_loss': best_loss,
'best_metric': best_metric,
'best_accuracy': best_accuracy,
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict()
}
checkpoint_filename = name + '-checkpoint-%s%s.pth' % (full_name, cycle)
checkpoint_file = checkpoint_dir / checkpoint_filename
torch.save(checkpoint, checkpoint_file)
checkpoint_file_simple = Path(checkpoint_dir / (name + '-checkpoint-%s%s.pth' % (args.data_fold, cycle)))
if checkpoint_file_simple.is_symlink() or checkpoint_file_simple.exists():
checkpoint_file_simple.unlink()
checkpoint_file_simple.symlink_to(checkpoint_file.relative_to(checkpoint_file_simple.parent))
if epoch_loss < best_loss:
best_loss = epoch_loss
save_checkpoint('best-loss')
if epoch_metric > best_metric:
best_metric = epoch_metric
save_checkpoint('best-metric')
if epoch_accuracy > best_accuracy:
best_accuracy = epoch_accuracy
save_checkpoint('best-accuracy')
save_checkpoint('last')
return epoch_loss, epoch_metric, epoch_accuracy
def train(self):
min_val_loss = np.inf
self.model.to(self.device) #gpu
for epoch in range(self.strt, self.epochs + self.strt):
self.scheduler.step() #decay lr if it hits milestones
train_loss = 0.0
self.model.train()
for i, batch in enumerate(self.train_loader):
x, y = enable_cuda(batch, self.device)
self.optimizer.zero_grad()
prediction = self.model(x)
#print(f'prediction : {prediction[0]} target {y[0]}')
#print(prediction.shape,angles.shape)
loss = (prediction - y).norm(
2).mean() if self.criterion == None else self.criterion(
prediction, y) #.unsqueeze(1))
loss.backward()
self.optimizer.step()
train_loss += loss.data.item()
if i % 100 == 0:
print(
f'Training loss at Epoch: {epoch} | Loss: {train_loss / (i + 1)}'
)
#validation
self.model.eval()
val_loss = 0.0
with torch.set_grad_enabled(False):
for i, batch in enumerate(self.val_loader):
x, y = enable_cuda(batch, self.device)
prediction = self.model(x)
#print(prediction.shape,angles.shape)
loss = (prediction - y).norm(2).mean(
) if self.criterion == None else self.criterion(
prediction, y) #.unsqueeze(1))
val_loss += loss.data.item()
if i % 100 == 0:
print(f'Validation Loss: {val_loss / (i + 1)}')
if val_loss < min_val_loss:
min_val_loss = val_loss
state = {
'epoch': epoch + 1,
'state_dict': self.model.state_dict(),
'optimizer': self.optimizer.state_dict(),
'scheduler': self.scheduler.state_dict(),
}
self.save(state, "best")
elif epoch % 5 == 0 or epoch == self.epochs + self.strt - 1:
min_val_loss = val_loss
state = {
'epoch': epoch + 1,
'state_dict': self.model.state_dict(),
'optimizer': self.optimizer.state_dict(),
'scheduler': self.scheduler.state_dict(),
}
self.save(state, "")
dtdata += time.time() - tdata
tforward = time.time()
print "coord-shape: flow1=", coord_t.shape
print "src feats-shape: ", srcpix_t.shape
print "tarpix_flow1 feats-shape: ", tarpix_flow1_t.shape
if tarpix_flow2_t is not None:
print "tarpix_flow2 feats-shape: ", tarpix_flow2_t.shape
if truth_flow1_t is not None:
print "truth flow1: ", truth_flow1_t.shape
if truth_flow2_t is not None:
print "truth flow2: ", truth_flow2_t.shape
with torch.autograd.profiler.profile(enabled=ENABLE_PROFILER,
use_cuda=PROF_USE_CUDA) as prof:
# workup
with torch.set_grad_enabled(run_w_grad):
predict1_t, predict2_t = model.forward(coord_t, srcpix_t,
tarpix_flow1_t,
tarpix_flow2_t,
batchsize)
dtforward += time.time() - tforward
print "output: flow1=", predict1_t.features.shape, " predict2_t=", predict2_t.features.shape
if test_loss:
tloss = time.time()
loss, flow1loss, flow2loss = consistency_loss(
coord_t, predict1_t, predict2_t, truth_flow1_t, truth_flow2_t)
print "loss: tot=", loss.detach().cpu().item()
if flow1loss is not None:
print " flow1=", flow1loss.detach().cpu().item()
def __enter__(self):
torch.set_grad_enabled(False)
def run(split, upto=None):
torch.set_grad_enabled(split=='train')
model.train() if split == 'train' else model.eval()
nsamples = 1 if split == 'train' else xte
N, D = x.size()
B = 128
n_steps = N // B if upto is None else min(N//B, upto)
losses = []
for step in range(n_steps):
xb = Variable(x[step * B: step * B + B])
xbhat = torch.zeros_like(xb)
for s in range(nsamples):
if step % args.resample_every == 0 or split == 'test':
model.update_masks()
xbhat += model(xb)
xbhat /= nsamples
loss = F.binary_cross_entropy_with_logits(xbhat, xb, size_average=False) / B
lossf = loss.data.item()
losses.append(lossf)
if split == 'train':
opt.zero_grad()
loss.backward()
opt.step()
print("%s epoch avg loss: %f" %(split, np.mean(losses)))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--data-path', required=True, type=str, help="Path to binarized_mnist.npz")
parser.add_argument('-q', '--hiddens', type=str, default='500', help="Comma separated sizes for hidden layers, e.g. 500, or 500,500")
parser.add_argument('-n', '--num-masks', type=int, default=1, help="Number of orderings for order/connection-agnostic training")
parser.add_argument('-r', '--resample-every', type=int, default=20, help="For efficiency we can choose to resample orders/masks only once every this many steps")
parser.add_argument('-s', '--samples', type=int, default=1, help="How many samples of connectivity/masks to average logits over during inference")
args = parser.parse_args()
np.random_seed(42)
torch.manual_seed(42)
torch.cuda.manual_seed_all(42)
print("loading binarized mnist from", args.data_path)
mnist = np.load(args.data_path)
xtr, xte = mnist['train_data'], mnist['valid_data']
xtr = torch.from_numpy(xtr).cuda()
xte = torch.from_numpy(xte).cuda()
# construct model and ship to GPU
hidden_list = list(map(int, args.hiddens.split(',')))
model = MADE(xtr.size(1), hidden_list, xtr.size(1), num_masks=args.num_masks)
print("number of model parameters:",sum([np.prod(p.size()) for p in model.parameters()]))
model.cuda()
# set up the optimizer
opt = torch.optim.Adam(model.parameters(), 1e-3, weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=45, gamma=0.1)
# start the training
for epoch in range(100):
print("epoch %d" % (epoch, ))
scheduler.step(epoch)
run_epoch('test', upto=5) # run only a few batches for approximate test accuracy
run_epoch('train')
print("optimization done. full test set eval:")
run_epoch('test')
def start(data_type):
def check_grad():
print('=========================GRAD CHECK===========================')
for name,param in model.named_parameters():
print(name)
print(f'value:{param.data}')
print(f'gradient:{param.grad}')
def complex_matmul(A1,B1,A2,B2):
real_part = torch.matmul(A1,A2)-torch.matmul(B1,B2)
imag_part = torch.matmul(A1,B2)+torch.matmul(B1,A2)
return [real_part, imag_part]
def forward(loader):
total_loss = []
for data in tqdm(loader):
data = data.to(device)
if data_type == 'w':
out = model(data).view(-1,32,200)
out = out * data.s.unsqueeze(-1)
out_real = out[...,:100].unsqueeze(-2)
out_imag = out[...,100:].unsqueeze(-2)
w_real = data.w[...,:100].unsqueeze(-2)
w_imag = data.w[...,100:].unsqueeze(-2)
out_ = complex_matmul(out_real, out_imag, pole_matrix_real, pole_matrix_imag)
w_ = complex_matmul(w_real, w_imag, pole_matrix_real, pole_matrix_imag)
loss = F.l1_loss(out_[0], w_[0]) + F.l1_loss(out_[1],w_[1])
elif data_type == 'dis':
out = model(data)
out = out * data.s[data.batch]
loss = F.l1_loss(out, data.dis)
elif data_type == 's':
out = model(data).reshape(-1,2)
loss = F.cross_entropy(out, data.s.reshape(-1))
if torch.is_grad_enabled():
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_loss.append(loss.item())
return np.mean(total_loss)
def get_loader(phase, bz = 10):
transforms = [RandomSample(2048)]
if data_type == 'dis':
transforms += [RandomRotate(180,0),RandomRotate(180,1),RandomRotate(180,2)]
return DataLoader(ModalDataset(args.dataset + phase, transforms = transforms),batch_size=bz,num_workers=10)
#==========================initialize================================
device = torch.device(f'cuda:{args.cuda}')
if data_type == 'w':
model = dgcnn_classification(32*200).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.8)
pole_matrix = np.load('weights/pole_matrixs.npy')
pole_matrix = np.transpose(pole_matrix, (0,2,1))
pole_matrix_real = torch.FloatTensor(pole_matrix.real).to(device)
pole_matrix_imag = torch.FloatTensor(pole_matrix.imag).to(device)
elif data_type == 'dis':
model = dgcnn_segmentation(32).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.8)
elif data_type == 's':
model = dgcnn_segmentation(32*2).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.8)
loader = {
'train': get_loader('train'),
'test' : get_loader('test')
}
torch.set_grad_enabled(False)
#==========================train=====================================
for epoch in range(200):
model.train()
print(f'============={epoch}=============')
with torch.set_grad_enabled(True):
print('train loss:{:.5f}'.format(forward(loader['train'])))
model.eval()
print('test loss:{:.5f}'.format(forward(loader['test'])))
torch.save(model.state_dict(), f'weights/{data_type}_weights.pt')
scheduler.step()
def ddpg_learner(process_ind, args,
global_logs,
learner_logs,
model_prototype,
global_memory,
global_model,
global_optimizer):
# logs
print("---------------------------->", process_ind, "learner")
# env
# memory
# model
local_device = torch.device('cuda:' + str(args.gpu_ind))
# global_device = torch.device('cpu')
# local_model = model_prototype(args.model_params,
# args.state_shape,
# args.action_space,
# args.action_shape).to(local_device)
local_target_model = model_prototype(args.model_params,
args.norm_val,
args.state_shape,
args.action_space,
args.action_shape).to(local_device)
# # sync global model to local
# local_model.load_state_dict(global_model.state_dict())
# update_target_model(local_model, local_target_model) # do a hard update in the beginning
update_target_model(global_model, local_target_model) # do a hard update in the beginning
# optimizer
local_optimizer = args.agent_params.optim(global_model.parameters(),
lr=args.agent_params.lr,
weight_decay=args.agent_params.weight_decay)
# params
# setup
# local_model.train()
global_model.train()
torch.set_grad_enabled(True)
# main control loop
step = 0
while global_logs.learner_step.value < args.agent_params.steps:
if global_memory.size > args.agent_params.learn_start:
# # sync global model to local
# local_model.load_state_dict(global_model.state_dict())
# sample batch from global_memory
experiences = global_memory.sample(args.agent_params.batch_size)
state0s, actions, rewards, gamma1s, state1s, terminal1s = experiences
# learn on this batch - setup
# global_optimizer.zero_grad()
# state0s = state0s.to(local_device)
# state1s = state1s.to(local_device)
local_optimizer.zero_grad()
state0s = state0s.cuda(non_blocking=True).to(local_device)
state1s = state1s.cuda(non_blocking=True).to(local_device)
# learn on this batch - actor loss
# _, qvalues = local_model(state0s)
_, qvalues = global_model(state0s)
actor_loss = - qvalues.mean()
# local_model.actor.zero_grad()
actor_loss.backward()
# nn.utils.clip_grad_value_(local_model.actor.parameters(), args.agent_params.clip_grad)
nn.utils.clip_grad_value_(global_model.actor.parameters(), args.agent_params.clip_grad)
# learn on this batch - critic loss
_, target_qvalues = local_target_model(state1s)
target_qvalues = rewards.to(local_device) + gamma1s.to(local_device) * target_qvalues.detach() * (1 - terminal1s.to(local_device))
# predict_qvalues = local_model.forward_critic(state0s, actions.to(local_device))
predict_qvalues = global_model.forward_critic(state0s, actions.to(local_device))
critic_loss = args.agent_params.value_criteria(predict_qvalues, target_qvalues)
# local_model.critic.zero_grad()
critic_loss.backward()
# nn.utils.clip_grad_value_(local_model.critic.parameters(), args.agent_params.clip_grad)
nn.utils.clip_grad_value_(global_model.critic.parameters(), args.agent_params.clip_grad)
# learn on this batch - sync local grads to global
# ensure_global_grads(local_model, global_model, local_device, global_device)
# global_optimizer.step()
local_optimizer.step()
# update target_model
# update_target_model(local_model, local_target_model, args.agent_params.target_model_update, step)
update_target_model(global_model, local_target_model, args.agent_params.target_model_update, step)
# update counters
with global_logs.learner_step.get_lock():
global_logs.learner_step.value += 1
step += 1
# report stats
if step % args.agent_params.learner_freq == 0: # then push local stats to logger & reset local
learner_logs.actor_loss.value += actor_loss.item()
learner_logs.critic_loss.value += critic_loss.item()
learner_logs.loss_counter.value += 1
else: # wait for memory_size to be larger than learn_start
time.sleep(1)
import PIL.Image
import sys
try:
from .correlation import correlation # the custom cost volume layer
except:
sys.path.insert(0, './correlation')
import correlation # you should consider upgrading python
# end
##########################################################
assert (int(str('').join(torch.__version__.split('.')[0:2])) >= 13
) # requires at least pytorch version 1.3.0
torch.set_grad_enabled(
False) # make sure to not compute gradients for computational performance
torch.backends.cudnn.enabled = True # make sure to use cudnn for computational performance
##########################################################
arguments_strModel = 'css'
arguments_strFirst = './images/first.png'
arguments_strSecond = './images/second.png'
arguments_strOut = './out.flo'
for strOption, strArgument in getopt.getopt(
sys.argv[1:], '',
[strParameter[2:] + '=' for strParameter in sys.argv[1::2]])[0]:
if strOption == '--model' and strArgument != '':
arguments_strModel = strArgument # which model to use
def trainResnet(model,
dataloaders,
criterion,
optimizer,
num_epochs=25,
is_inception=False):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
since = time.time()
val_acc_history = []
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(dataloaders[phase].dataset)
epoch_acc = running_corrects.double() / len(
dataloaders[phase].dataset)
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
epoch_acc))
# deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
fName = '%s_best_rank.pth.tar' % ('resnet_50')
torch.save(
{
'epoch': epoch,
'arch': 'resnet',
'state_dict': best_model_wts
}, fName)
if phase == 'val':
val_acc_history.append(epoch_acc)
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
return model, val_acc_history
def train(args, model, criterion, optimizer, device, train_dataloader, writer,
epoch):
torch.set_grad_enabled(True)
model.train()
running_loss = 0.0
correct = 0
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
losses_contrast_frame = AverageMeter()
losses_contrast_clip = AverageMeter()
losses_contrast = AverageMeter()
losses_order = AverageMeter()
end = time.time()
torch.cuda.empty_cache()
for i, data in enumerate(train_dataloader, 1):
data_time.update(time.time() - end)
# get inputs
#tuple_clips, tuple_orders, tuple_clips_random, tuple_orders_random, index = data
tuple_clips, tuple_orders, index = data
bsz = tuple_clips.size(0)
inputs = tuple_clips.to(device)
#inputs_random = tuple_clips_random.to(device)
targets = [order_class_index(order) for order in tuple_orders]
targets = torch.tensor(targets).to(device)
index = index.to(device)
# zero the parameter gradients
#optimizer.zero_grad()
# forward and backward
contrast_loss_1, contrast_loss_2, contrast_loss_3, contrast_loss_clip, outputs = model(
inputs, tuple_orders) # return logits here
loss_contrast_frame = (contrast_loss_1.sum() + contrast_loss_2.sum() +
contrast_loss_3.sum()) / (3 * args.bs)
loss_contrast_clip = contrast_loss_clip.sum() / args.bs
loss_order = criterion(outputs, targets)
loss = args.weight_contrast_frame * loss_contrast_frame + args.weight_contrast_clip * loss_contrast_clip + args.weight_order * loss_order
#loss=args.weight_order*loss_order
# ===================backward=====================
optimizer.zero_grad()
loss.backward()
optimizer.step()
# ===================meters=====================
#losses_contrast.update(args.weight_contrast*loss_contrast.item(), bsz)
losses_contrast_frame.update(
args.weight_contrast_frame * loss_contrast_frame.item(), bsz)
losses_contrast_clip.update(
args.weight_contrast_clip * loss_contrast_clip.item(), bsz)
losses_order.update(args.weight_order * loss_order.item(), bsz)
losses.update(loss.item(), bsz)
batch_time.update(time.time() - end)
end = time.time()
# compute loss and acc
running_loss += loss.item()
pts = torch.argmax(outputs, dim=1)
correct += torch.sum(targets == pts).item()
# print statistics and write summary every N batch
# print info
if i % 20 == 0:
log_str = (
'Train: [{0}/{1}][{2}/{3}] '
#'BT {batch_time.val:.3f} ({batch_time.avg:.3f}) '
#'DT {data_time.val:.3f} ({data_time.avg:.3f}) '
'loss_contrast_frame {loss_contrast_frame.val:.3f} ({loss_contrast_frame.avg:.3f}) '
'loss_contrast_clip {loss_contrast_clip.val:.3f} ({loss_contrast_clip.avg:.3f}) '
'loss_order {loss_order.val:.3f} ({loss_order.avg:.3f}) '
'loss {loss.val:.3f} ({loss.avg:.3f})'.format(
epoch,
args.epochs,
i,
len(train_dataloader),
loss_contrast_frame=losses_contrast_frame,
loss_contrast_clip=losses_contrast_clip,
loss_order=losses_order,
loss=losses))
logging.info(log_str)
if i % args.pf == 0:
avg_loss = running_loss / args.pf
avg_acc = correct / (args.pf * args.bs)
logging.info(
'[TRAIN] epoch-{}, batch-{}, loss: {:.3f}, acc: {:.3f}'.format(
epoch, i, avg_loss, avg_acc))
#step = (epoch-1)*len(train_dataloader) + i
#writer.add_scalar('train/CrossEntropyLoss', avg_loss, step)
#writer.add_scalar('train/Accuracy', avg_acc, step)
running_loss = 0.0
correct = 0
# summary params and grads per eopch
#for name, param in model.named_parameters():
# writer.add_histogram('params/{}'.format(name), param, epoch)
# writer.add_histogram('grads/{}'.format(name), param.grad, epoch)
avg_loss = running_loss / len(train_dataloader)
return avg_loss
def train_eval_model(model,
criterion,
optimizer,
dataloader,
num_epochs,
resume=False,
start_epoch=0):
print("Start training...")
since = time.time()
dataloader["train"].dataset.set_num_graphs(
cfg.TRAIN.num_graphs_in_matching_instance)
dataset_size = len(dataloader["train"].dataset)
device = next(model.parameters()).device
print("model on device: {}".format(device))
checkpoint_path = Path(cfg.model_dir) / "params"
if not checkpoint_path.exists():
checkpoint_path.mkdir(parents=True)
if resume:
params_path = os.path.join(cfg.warmstart_path, f"params.pt")
print("Loading model parameters from {}".format(params_path))
model.load_state_dict(torch.load(params_path))
optim_path = os.path.join(cfg.warmstart_path, f"optim.pt")
print("Loading optimizer state from {}".format(optim_path))
optimizer.load_state_dict(torch.load(optim_path))
# Evaluation only
if cfg.evaluate_only:
assert resume
print(f"Evaluating without training...")
accs, f1_scores = eval_model(model, dataloader["test"], verbose=True)
acc_dict = {
"acc_{}".format(cls): single_acc
for cls, single_acc in zip(dataloader["train"].dataset.classes,
accs)
}
f1_dict = {
"f1_{}".format(cls): single_f1_score
for cls, single_f1_score in zip(
dataloader["train"].dataset.classes, f1_scores)
}
acc_dict.update(f1_dict)
acc_dict["matching_accuracy"] = torch.mean(accs)
acc_dict["f1_score"] = torch.mean(f1_scores)
time_elapsed = time.time() - since
print("Evaluation complete in {:.0f}h {:.0f}m {:.0f}s".format(
time_elapsed // 3600, (time_elapsed // 60) % 60,
time_elapsed % 60))
return model, acc_dict
_, lr_milestones, lr_decay = lr_schedules[cfg.TRAIN.lr_schedule]
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=lr_milestones,
gamma=lr_decay)
for epoch in range(start_epoch, num_epochs):
print("Epoch {}/{}".format(epoch, num_epochs - 1))
print("-" * 10)
model.train() # Set model to training mode
print("lr = " + ", ".join(
["{:.2e}".format(x["lr"]) for x in optimizer.param_groups]))
epoch_loss = 0.0
running_loss = 0.0
running_acc = 0.0
epoch_acc = 0.0
running_f1 = 0.0
epoch_f1 = 0.0
running_since = time.time()
iter_num = 0
# Iterate over data.
for inputs in dataloader["train"]:
data_list = [_.cuda() for _ in inputs["images"]]
points_gt_list = [_.cuda() for _ in inputs["Ps"]]
n_points_gt_list = [_.cuda() for _ in inputs["ns"]]
edges_list = [_.to("cuda") for _ in inputs["edges"]]
perm_mat_list = [
perm_mat.cuda() for perm_mat in inputs["gt_perm_mat"]
]
n_label_gt_list = [_.to("cuda") for _ in inputs["n_label"]]
c_label_gt_list = [_.to("cuda") for _ in inputs["c_label"]]
iter_num = iter_num + 1
# zero the parameter gradients
optimizer.zero_grad()
with torch.set_grad_enabled(True):
# forward
if cfg.CLASSIFICATION.cls_type is not None:
s_pred_list = model(data_list, points_gt_list, edges_list,
n_points_gt_list, perm_mat_list,
n_label_gt_list, c_label_gt_list)
if cfg.CLASSIFICATION.cls_type == "class":
# TODO
pass
else:
raise NotImplementedError(
"No such mode in classification")
else:
s_pred_list = model(data_list, points_gt_list, edges_list,
n_points_gt_list, perm_mat_list)
loss = sum([
criterion(s_pred, perm_mat)
for s_pred, perm_mat in zip(s_pred_list, perm_mat_list)
])
loss /= len(s_pred_list)
# backward + optimize
loss.backward()
optimizer.step()
tp, fp, fn = get_pos_neg_from_lists(s_pred_list, perm_mat_list)
f1 = f1_score(tp, fp, fn)
acc, _, __ = matching_accuracy_from_lists(
s_pred_list, perm_mat_list)
# statistics
bs = perm_mat_list[0].size(0)
running_loss += loss.item() * bs # multiply with batch size
epoch_loss += loss.item() * bs
running_acc += acc.item() * bs
epoch_acc += acc.item() * bs
running_f1 += f1.item() * bs
epoch_f1 += f1.item() * bs
if iter_num % cfg.STATISTIC_STEP == 0:
running_speed = cfg.STATISTIC_STEP * bs / (time.time() -
running_since)
loss_avg = running_loss / cfg.STATISTIC_STEP / bs
acc_avg = running_acc / cfg.STATISTIC_STEP / bs
f1_avg = running_f1 / cfg.STATISTIC_STEP / bs
print(
"Epoch {:<4} Iter {:<4} {:>4.2f}sample/s Loss={:<8.4f} Accuracy={:<2.3} F1={:<2.3}"
.format(epoch, iter_num, running_speed, loss_avg,
acc_avg, f1_avg))
running_acc = 0.0
running_f1 = 0.0
running_loss = 0.0
running_since = time.time()
epoch_loss = epoch_loss / dataset_size
epoch_acc = epoch_acc / dataset_size
epoch_f1 = epoch_f1 / dataset_size
if cfg.save_checkpoint:
base_path = Path(checkpoint_path / "{:04}".format(epoch + 1))
Path(base_path).mkdir(parents=True, exist_ok=True)
path = str(base_path / "params.pt")
torch.save(model.state_dict(), path)
torch.save(optimizer.state_dict(), str(base_path / "optim.pt"))
print(
"Over whole epoch {:<4} -------- Loss: {:.4f} Accuracy: {:.3f} F1: {:.3f}"
.format(epoch, epoch_loss, epoch_acc, epoch_f1))
print()
# Eval in each epoch
accs, f1_scores = eval_model(model, dataloader["test"])
acc_dict = {
"acc_{}".format(cls): single_acc
for cls, single_acc in zip(dataloader["train"].dataset.classes,
accs)
}
f1_dict = {
"f1_{}".format(cls): single_f1_score
for cls, single_f1_score in zip(
dataloader["train"].dataset.classes, f1_scores)
}
acc_dict.update(f1_dict)
acc_dict["matching_accuracy"] = torch.mean(accs)
acc_dict["f1_score"] = torch.mean(f1_scores)
scheduler.step()
time_elapsed = time.time() - since
print("Training complete in {:.0f}h {:.0f}m {:.0f}s".format(
time_elapsed // 3600, (time_elapsed // 60) % 60, time_elapsed % 60))
return model, acc_dict
pretrained_dict = torch.load(
pretrained_path,
map_location=lambda storage, loc: storage.cuda(device))
if "state_dict" in pretrained_dict.keys():
pretrained_dict = remove_prefix(pretrained_dict['state_dict'],
'module.')
else:
pretrained_dict = remove_prefix(pretrained_dict, 'module.')
check_keys(model, pretrained_dict)
model.load_state_dict(pretrained_dict, strict=False)
return model
if __name__ == '__main__':
torch.set_grad_enabled(False)
# net and model
net = YuFaceDetectNet(phase='test', size=None) # initialize detector
net = load_model(net, args.trained_model, True)
net.eval()
print('Finished loading model!')
bs = args.input_batch_size
h = args.input_height
w = args.input_width
img = torch.randn(bs, 3, h, w, requires_grad=False)
img = img.to(torch.device('cpu'))
input_names = ['input']
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import math
import time
import tqdm
import os
import argparse
import copy
import datetime
### My libs
from dataset import TIANCHI
from models.model3 import STM
torch.set_grad_enabled(False) # Volatile
def get_arguments():
parser = argparse.ArgumentParser(description="SST")
parser.add_argument("-g", type=str, help="0; 0,1; 0,3; etc", required=True)
parser.add_argument("-s", type=str, help="set", required=True)
parser.add_argument("-viz", help="Save visualization", action="store_true")
parser.add_argument("-D", type=str, help="path to data", default='/local/DATA')
parser.add_argument("-M", type=str, help="path to model", default='/local/model')
return parser.parse_args()
args = get_arguments()
GPU = args.g
def train_age(kfold, batchsize, lr_age, lr_gender, num_epochs, p_augment,
device, num_age_classes, num_gender_classes, test_fold,
train_fold, random_seed):
all_accuracy_age = []
all_val_loss_age = []
all_stat_fold = []
for fold in range(kfold):
all_stat = defaultdict(list)
# image paths
train_data = train_fold[fold]['image_path'].copy().reset_index(
drop=True).to_list()
test_data = test_fold[fold]['image_path'].copy().reset_index(
drop=True).to_list()
#get label
train_age_label = train_fold[fold]['age'].copy().reset_index(
drop=True).to_list()
train_gender_label = train_fold[fold]['gender'].copy().reset_index(
drop=True).to_list()
test_age_label = test_fold[fold]['age'].copy().reset_index(
drop=True).to_list()
test_gender_label = test_fold[fold]['gender'].copy().reset_index(
drop=True).to_list()
#create train-validation stratified split
sss = StratifiedShuffleSplit(n_splits=10, random_state=random_seed)
#split based on age, more balanced for both age and gender
train_idx, val_idx = list(sss.split(train_data, train_age_label))[0]
train_idx = list(train_idx)
val_idx = list(val_idx)
#create dataloader for gender
train_dataset = AgeDataset(
'',
list(np.array(train_data)[train_idx]),
list(np.array(train_age_label)[train_idx]),
list(np.array(train_gender_label)[train_idx]),
p_augment=p_augment)
val_dataset = AgeDataset('',
list(np.array(train_data)[val_idx]),
list(np.array(train_age_label)[val_idx]),
list(np.array(train_gender_label)[val_idx]),
validation=True)
test_dataset = AgeDataset('',
test_data,
test_age_label,
test_gender_label,
validation=True)
train_loader = DataLoader(train_dataset,
batch_size=batchsize,
shuffle=True)
val_loader = DataLoader(val_dataset,
batch_size=batchsize,
shuffle=False)
test_loader = DataLoader(test_dataset,
batch_size=batchsize,
shuffle=False)
val_gender_label = list(np.array(train_gender_label)[val_idx])
val_age_label = list(np.array(train_age_label)[val_idx])
model = InceptionResnetV1(classify=True,
pretrained='vggface2',
num_classes=num_age_classes)
model = model.to(device)
#optimizer
optimizer = optim.AdamW(model.parameters(), lr=lr_age)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [5, 10])
#loss
criterion = nn.CrossEntropyLoss()
best_acc_age = 0
best_val_loss_age = 999
print(f'Fold {fold+1}\n')
for epoch in range(num_epochs):
print(f'epoch: {epoch}\n')
train_loss_age = 0
val_loss_age = 0
#Training
model.train()
iterat = 0
vsego = len(train_loader)
for batch in train_loader:
print(f'batch_num: {100*(iterat/vsego)}%\n')
# Load image batch
batch_data, batch_age_label = batch
batch_data = batch_data.to(device)
batch_age_label = batch_age_label.to(device)
# Clear gradients
optimizer.zero_grad()
with torch.set_grad_enabled(True):
pred_age = model(batch_data)
loss_age = criterion(pred_age, batch_age_label)
train_loss_age += loss_age.detach().item()
loss_age.backward()
optimizer.step()
iterat = iterat + 1
#Validation
model.eval()
all_pred_age = torch.empty(0).to(device)
for batch in val_loader:
# Load image batch
batch_data, batch_age_label = batch
batch_data = batch_data.to(device)
batch_age_label = batch_age_label.to(device)
with torch.set_grad_enabled(False):
pred_age = model(batch_data)
loss_age = criterion(pred_age, batch_age_label)
val_loss_age += loss_age.detach().item()
all_pred_age = torch.cat(
(all_pred_age,
nn.functional.softmax(pred_age.detach(), dim=1)), 0)
train_loss_age /= len(train_loader)
val_loss_age /= len(val_loader)
all_pred_age = all_pred_age.cpu().numpy()
pred_label_age = list(np.argmax(all_pred_age, axis=1))
acc_age = accuracy_score(val_age_label, pred_label_age)
if acc_age > best_acc_age:
best_acc_age = acc_age
best_val_loss_age = val_loss_age
torch.save(model.state_dict(), f'models/age_model{fold}.pth')
all_stat['train_loss'].append(train_loss_age)
all_stat['val_loss'].append(val_loss_age)
all_stat['val_acc'].append(acc_age)
print(
f'Epoch {epoch} | train loss: {train_loss_age} | val loss: {val_loss_age} | accuracy: {round(acc_age*100, 2)}%'
)
scheduler.step()
#INFERENCE
with torch.no_grad():
model.load_state_dict(torch.load(f'models/age_model{fold}.pth'))
model.eval()
test_pred_age = torch.empty(0).to(device)
for batch in test_loader:
# Load image batch
batch_data, batch_age_label = batch
batch_data = batch_data.to(device)
batch_age_label = batch_age_label.to(device)
with torch.set_grad_enabled(False):
pred_age = model(batch_data)
test_pred_age = torch.cat(
(test_pred_age,
nn.functional.softmax(pred_age.detach(), dim=1)), 0)
test_pred_age = test_pred_age.cpu().numpy()
pred_label_age = list(np.argmax(test_pred_age, axis=1))
acc_age = accuracy_score(test_age_label, pred_label_age)
all_stat['test_acc'].append(acc_age)
all_stat['conf'].append(
confusion_matrix(test_age_label,
pred_label_age,
labels=list(range(num_age_classes))))
all_stat['conf_norm'].append(
confusion_matrix(test_age_label,
pred_label_age,
normalize='true',
labels=list(range(num_age_classes))))
all_stat['test_pred'].append(pred_label_age)
all_stat['test_target'].append(test_age_label)
all_accuracy_age.append(acc_age)
all_val_loss_age.append(best_val_loss_age)
print(
f'TEST ACCURACY: {round(acc_age*100,2)}% | Val. Accuracy: {round(best_acc_age*100,2)}% | Val. Loss.: {best_val_loss_age}\n'
)
all_stat_fold.append(all_stat)
all_accuracy_age = np.array(all_accuracy_age)
all_val_loss_age = np.array(all_val_loss_age)
mean_accuracy_age = round(all_accuracy_age.mean() * 100, 2)
print(f'\nOverall Accuracy: {mean_accuracy_age} p/m')
def trans_from_files_beam(trans_source, trans_target, train_source, train_target,
embedding_params, encdec_params, seed=3, beam_size=10, use_attenMatrix=False):
batchSize = 1
beamSize = beam_size
print('trans_source :' + trans_source)
print('train_source :' + train_source)
print('train_target :' + train_target)
print('embedding_params :' + embedding_params)
print('encdec_params :' + encdec_params)
corpus = Corpus(train_source, train_source, train_target, trans_source,
trans_source, trans_target, minFreqSource, minFreqTarget, maxLen)
print('Source vocabulary size: ' + str(corpus.sourceVoc.size()))
print('Target vocabulary size: ' + str(corpus.targetVoc.size()))
print('# of training samples: ' + str(len(corpus.trainData)))
print('# of develop samples: ' + str(len(corpus.devData)))
print('SEED: ', str(seed))
embedding = Embedding(sourceEmbedDim, targetEmbedDim,
corpus.sourceVoc.size(), corpus.targetVoc.size())
encdec = EncDec(sourceEmbedDim, targetEmbedDim, hiddenDim,
corpus.targetVoc.size(), dropoutRate=dropoutRate, numLayers=numLayers)
encdec.wordPredictor.softmaxLayer.weight = embedding.targetEmbedding.weight
encdec.wordPredictor = nn.DataParallel(encdec.wordPredictor, gpuId)
batchListTrain = utils.buildBatchList(len(corpus.trainData), batchSize)
batchListDev = utils.buildBatchList(len(corpus.devData), batchSize)
withoutWeightDecay = []
withWeightDecay = []
for name, param in list(embedding.named_parameters()) + list(encdec.named_parameters()):
if 'bias' in name or 'Embedding' in name:
withoutWeightDecay += [param]
elif 'softmax' not in name:
withWeightDecay += [param]
optParams = [{'params': withWeightDecay, 'weight_decay': weightDecay},
{'params': withoutWeightDecay, 'weight_decay': 0.0}]
totalParamsNMT = withoutWeightDecay + withWeightDecay
opt = optim.SGD(optParams, momentum=momentumRate, lr=learningRate)
bestDevGleu = -1.0
prevDevGleu = -1.0
torch.set_grad_enabled(False)
embedding.load_state_dict(torch.load(embedding_params))
encdec.load_state_dict(torch.load(encdec_params))
embedding.to(device)
encdec.to(device)
embedding.eval()
encdec.eval()
f_trans = open('./trans.txt', 'w')
f_gold = open('./gold.txt', 'w')
f_attn = open('./attn.txt', 'w')
devPerp = 0.0
totalTokenCount = 0.0
attention_scores_matrix_list = []
trans_result = []
for batch in tqdm(batchListDev):
batchSize = batch[1] - batch[0] + 1
batchInputSource, lengthsSource, batchInputTarget, batchTarget, lengthsTarget, tokenCount, batchData = corpus.processBatchInfoNMT(
batch, train=False, device=device)
inputSource = embedding.getBatchedSourceEmbedding(batchInputSource)
sourceH, (hn, cn) = encdec.encode(inputSource, lengthsSource)
if beamSize == 1:
indicesGreedy, lengthsGreedy, attentionIndices = encdec.greedyTrans(
corpus.targetVoc.bosIndex, corpus.targetVoc.eosIndex, lengthsSource, embedding.targetEmbedding, sourceH, (hn, cn), device, maxGenLen=maxLen)
else:
if use_attenMatrix == True:
indicesGreedy, lengthsGreedy, attentionIndices, attenMatrix = encdec.beamSearchAttn(
corpus.targetVoc.bosIndex, corpus.targetVoc.eosIndex, lengthsSource, embedding.targetEmbedding, sourceH, (hn, cn), device, beamSize=beamSize, maxGenLen=maxLen)
attention_scores_matrix_list.append(attenMatrix)
else:
indicesGreedy, lengthsGreedy, attentionIndices = encdec.beamSearch(
corpus.targetVoc.bosIndex, corpus.targetVoc.eosIndex, lengthsSource, embedding.targetEmbedding, sourceH, (hn, cn), device, beamSize=beamSize, maxGenLen=maxLen)
indicesGreedy = indicesGreedy.to(cpu)
for i in range(batchSize):
target_tokens = []
for k in range(lengthsGreedy[i] - 1):
index = indicesGreedy[i, k].item()
if index == corpus.targetVoc.unkIndex:
index = attentionIndices[i, k].item()
target_tokens.append(
batchData[i].sourceOrigStr[index] + ' ')
f_trans.write(batchData[i].sourceOrigStr[index] + ' ')
else:
f_trans.write(corpus.targetVoc.tokenList[index].str + ' ')
target_tokens.append(
corpus.targetVoc.tokenList[index].str + ' ')
trans_result.append(" ".join(target_tokens))
f_trans.write('\n')
for k in range(lengthsTarget[i] - 1):
index = batchInputTarget[i, k + 1].item()
if index == corpus.targetVoc.unkIndex:
f_gold.write(batchData[i].targetUnkMap[k] + ' ')
else:
f_gold.write(corpus.targetVoc.tokenList[index].str + ' ')
f_gold.write('\n')
# This is code to save attention max index.
for k in range(lengthsGreedy[i] - 1):
f_attn.write(str(attentionIndices[i, k].item()) + ' ')
f_attn.write('\n')
batchInputTarget = batchInputTarget.to(device)
batchTarget = batchTarget.to(device)
inputTarget = embedding.getBatchedTargetEmbedding(batchInputTarget)
loss = encdec(inputTarget, lengthsTarget, lengthsSource,
(hn, cn), sourceH, batchTarget)
loss = loss.sum()
devPerp += loss.item()
totalTokenCount += tokenCount
f_trans.close()
f_gold.close()
f_attn.close()
if use_attenMatrix == True:
return trans_result, attention_scores_matrix_list
def __enter__(self):
torch.set_grad_enabled(True)
def main():
# os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
assert torch.cuda.is_available(), "Currently, we only support CUDA version"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.cuda.set_device(device)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
random.seed(args.seed)
np.random.seed(args.seed)
Network = getattr(models, args.net) #
model = Network(**args.net_params)
model = torch.nn.DataParallel(model).to(device)
optimizer = getattr(torch.optim, args.opt)(model.parameters(), **args.opt_params)
criterion = getattr(criterions, args.criterion)
msg = ''
if args.resume:
if os.path.isfile(args.resume):
try:
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
print('Loaded checkpoint')
args.start_iter = checkpoint['iter']
print('Applying to model')
model.load_state_dict(checkpoint['state_dict'])
print('Applied to model')
if not args.finetune:
optimizer.load_state_dict(checkpoint['optim_dict'])
msg = ("=> loaded checkpoint '{}' (iter {})".format(args.resume, checkpoint['iter']))
except:
print("Raise EXCEPT: could not load checkpoint")
else:
msg = "=> no checkpoint found at '{}'".format(args.resume)
else:
msg = '-------------- New training session ----------------'
msg += '\n' + str(args)
logging.info(msg)
# Data loading code
Dataset = getattr(datasets, args.dataset) #
if args.prefix_path:
args.train_data_dir = os.path.join(args.prefix_path, args.train_data_dir)
train_list = os.path.join(args.train_data_dir, args.train_list)
train_set = Dataset(train_list, root=args.train_data_dir, for_train=True, transforms=args.train_transforms)
num_iters = args.num_iters or (len(train_set) * args.num_epochs) // args.batch_size
num_iters -= args.start_iter
train_sampler = CycleSampler(len(train_set), num_iters * args.batch_size)
train_loader = DataLoader(
dataset=train_set,
batch_size=args.batch_size,
collate_fn=train_set.collate,
sampler=train_sampler,
num_workers=args.workers,
pin_memory=True,
worker_init_fn=init_fn)
start = time.time()
enum_batches = len(train_set) / float(args.batch_size) # nums_batch per epoch
losses = AverageMeter()
torch.set_grad_enabled(True)
for i, data in enumerate(train_loader, args.start_iter):
elapsed_bsize = int(i / enum_batches) + 1
epoch = int((i + 1) / enum_batches)
setproctitle.setproctitle("Epoch:{}/{}".format(elapsed_bsize, args.num_epochs))
# actual training
adjust_learning_rate(optimizer, epoch, args.num_epochs, args.opt_params.lr)
data = [t.cuda(non_blocking=True) for t in data]
x, target = data[:2]
output = model(x)
if not args.weight_type: # compatible for the old version
args.weight_type = 'square'
if args.criterion_kwargs is not None:
loss = criterion(output, target, **args.criterion_kwargs)
else:
loss = criterion(output, target)
# measure accuracy and record loss
losses.update(loss.item(), target.numel())
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (i + 1) % int(enum_batches * args.save_freq) == 0 \
or (i + 1) % int(enum_batches * (args.num_epochs - 1)) == 0 \
or (i + 1) % int(enum_batches * (args.num_epochs - 2)) == 0 \
or (i + 1) % int(enum_batches * (args.num_epochs - 3)) == 0 \
or (i + 1) % int(enum_batches * (args.num_epochs - 4)) == 0:
file_name = os.path.join(ckpts, 'model_epoch_{}.pth'.format(epoch))
epoch_checkpoint = {
'iter': i,
'state_dict': model.state_dict(),
'optim_dict': optimizer.state_dict(),
}
torch.save(epoch_checkpoint, file_name)
if args.aws:
torch.save(epoch_checkpoint, os.path.join('/opt/ml/checkpoints', 'model_epoch_{}.pth'.format(epoch)))
# TODO: need to save to /opt/ml/checkpoints
msg = 'Iter {0:}, Epoch {1:.4f}, Loss {2:.7f}'.format(i + 1, (i + 1) / enum_batches, losses.avg)
logging.info(msg)
losses.reset()
i = num_iters + args.start_iter
file_name = os.path.join(ckpts, 'model_last.pth')
torch.save({
'iter': i,
'state_dict': model.state_dict(),
'optim_dict': optimizer.state_dict(),
},
file_name)
msg = 'total time: {:.4f} minutes'.format((time.time() - start) / 60)
logging.info(msg)
def main():
logger.info('------------MODEL TEST INPUT--------------')
logger.info('loading config file...')
global_config = read_config()
# set random seed
seed = global_config['global']['random_seed']
torch.manual_seed(seed)
torch.set_grad_enabled(False) # make sure all tensors below have require_grad=False
logger.info('reading squad dataset...')
dataset = SquadDataset(global_config)
logger.info('constructing model...')
model_choose = global_config['global']['model']
dataset_h5_path = global_config['data']['dataset_h5']
if model_choose == 'base':
model = BaseModel(dataset_h5_path,
model_config=read_config('config/base_model.yaml'))
elif model_choose == 'match-lstm':
model = MatchLSTM(dataset_h5_path)
elif model_choose == 'match-lstm+':
model = MatchLSTMPlus(dataset_h5_path)
elif model_choose == 'r-net':
model = RNet(dataset_h5_path)
elif model_choose == 'm-reader':
model = MReader(dataset_h5_path)
else:
raise ValueError('model "%s" in config file not recoginized' % model_choose)
model.eval() # let training = False, make sure right dropout
logging.info('model parameters count: %d' % count_parameters(model))
# load model weight
logger.info('loading model weight...')
model_weight_path = global_config['data']['model_path']
is_exist_model_weight = os.path.exists(model_weight_path)
assert is_exist_model_weight, "not found model weight file on '%s'" % model_weight_path
weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage)
model.load_state_dict(weight, strict=False)
# manual input qa
context = "In 1870, Tesla moved to Karlovac, to attend school at the Higher Real Gymnasium, where he was " \
"profoundly influenced by a math teacher Martin Sekuli\u0107.:32 The classes were held in German, " \
"as it was a school within the Austro-Hungarian Military Frontier. Tesla was able to perform integral " \
"calculus in his head, which prompted his teachers to believe that he was cheating. He finished a " \
"four-year term in three years, graduating in 1873.:33 "
question1 = "What language were classes held in at Tesla's school?"
answer1 = ["German"]
question2 = "Who was Tesla influenced by while in school?"
answer2 = ["Martin Sekuli\u0107"]
question3 = "Why did Tesla go to Karlovac?"
answer3 = ["attend school at the Higher Real Gymnasium", 'to attend school']
# change here to select questions
question = question1
answer = answer1[0]
# preprocess
nlp = spacy.load('en')
context_doc = DocText(nlp, context, global_config['preprocess'])
question_doc = DocText(nlp, question, global_config['preprocess'])
context_doc.update_em(question_doc)
question_doc.update_em(context_doc)
context_token = context_doc.token
question_token = question_doc.token
context_id_char = to_long_tensor(dataset.sentence_char2id(context_token))
question_id_char = to_long_tensor(dataset.sentence_char2id(question_token))
context_id, context_f = context_doc.to_id(dataset.meta_data)
question_id, question_f = question_doc.to_id(dataset.meta_data)
bat_input = [context_id, question_id, context_id_char, question_id_char, context_f, question_f]
bat_input = [x.unsqueeze(0) if x is not None else x for x in bat_input]
out_ans_prop, out_ans_range, vis_param = model.forward(*bat_input)
out_ans_range = out_ans_range.numpy()
start = out_ans_range[0][0]
end = out_ans_range[0][1] + 1
out_answer_id = context_id[start:end]
out_answer = dataset.sentence_id2word(out_answer_id)
logging.info('Predict Answer: ' + ' '.join(out_answer))
# to show on visdom
s = 0
e = 48
x_left = vis_param['match']['left']['alpha'][0, :, s:e].numpy()
x_right = vis_param['match']['right']['alpha'][0, :, s:e].numpy()
x_left_gated = vis_param['match']['left']['gated'][0, :, s:e].numpy()
x_right_gated = vis_param['match']['right']['gated'][0, :, s:e].numpy()
draw_heatmap_sea(x_left,
xlabels=context_token[s:e],
ylabels=question_token,
answer=answer,
save_path='data/test-left.png',
bottom=0.45)
draw_heatmap_sea(x_right,
xlabels=context_token[s:e],
ylabels=question_token,
answer=answer,
save_path='data/test-right.png',
bottom=0.45)
enable_self_match = False
if enable_self_match:
x_self_left = vis_param['self']['left']['alpha'][0, s:e, s:e].numpy()
x_self_right = vis_param['self']['right']['alpha'][0, s:e, s:e].numpy()
draw_heatmap_sea(x_self_left,
xlabels=context_token[s:e],
ylabels=context_token[s:e],
answer=answer,
save_path='data/test-self-left.png',
inches=(11, 11),
bottom=0.2)
draw_heatmap_sea(x_self_right,
xlabels=context_token[s:e],
ylabels=context_token[s:e],
answer=answer,
save_path='data/test-self-right.png',
inches=(11, 11),
bottom=0.2)
def train_helper(model: torchvision.models.resnet.ResNet,
dataloaders: Dict[str, torch.utils.data.DataLoader],
dataset_sizes: Dict[str,
int], criterion: torch.nn.modules.loss,
optimizer: torch.optim, scheduler: torch.optim.lr_scheduler,
num_epochs: int, writer: IO, device: torch.device,
start_epoch: int, batch_size: int, save_interval: int,
checkpoints_folder: Path, num_layers: int, classes: List[str],
num_classes: int) -> None:
"""
Function for training ResNet.
Args:
model: ResNet model for training.
dataloaders: Dataloaders for IO pipeline.
dataset_sizes: Sizes of the training and validation dataset.
criterion: Metric used for calculating loss.
optimizer: Optimizer to use for gradient descent.
scheduler: Scheduler to use for learning rate decay.
start_epoch: Starting epoch for training.
writer: Writer to write logging information.
device: Device to use for running model.
num_epochs: Total number of epochs to train for.
batch_size: Mini-batch size to use for training.
save_interval: Number of epochs between saving checkpoints.
checkpoints_folder: Directory to save model checkpoints to.
num_layers: Number of layers to use in the ResNet model from [18, 34, 50, 101, 152].
classes: Names of the classes in the dataset.
num_classes: Number of classes in the dataset.
"""
since = time.time()
# Initialize all the tensors to be used in training and validation.
# Do this outside the loop since it will be written over entirely at each
# epoch and doesn't need to be reallocated each time.
train_all_labels = torch.empty(size=(dataset_sizes["train"], ),
dtype=torch.long).cpu()
train_all_predicts = torch.empty(size=(dataset_sizes["train"], ),
dtype=torch.long).cpu()
val_all_labels = torch.empty(size=(dataset_sizes["val"], ),
dtype=torch.long).cpu()
val_all_predicts = torch.empty(size=(dataset_sizes["val"], ),
dtype=torch.long).cpu()
# Train for specified number of epochs.
for epoch in range(start_epoch, num_epochs):
# Training phase.
model.train(mode=True)
train_running_loss = 0.0
train_running_corrects = 0
# Train over all training data.
for idx, (inputs, labels) in enumerate(dataloaders["train"]):
train_inputs = inputs.to(device=device)
train_labels = labels.to(device=device)
optimizer.zero_grad()
# Forward and backpropagation.
with torch.set_grad_enabled(mode=True):
train_outputs = model(train_inputs)
__, train_preds = torch.max(train_outputs, dim=1)
train_loss = criterion(input=train_outputs,
target=train_labels)
train_loss.backward()
optimizer.step()
# Update training diagnostics.
train_running_loss += train_loss.item() * train_inputs.size(0)
train_running_corrects += torch.sum(
train_preds == train_labels.data, dtype=torch.double)
start = idx * batch_size
end = start + batch_size
train_all_labels[start:end] = train_labels.detach().cpu()
train_all_predicts[start:end] = train_preds.detach().cpu()
calculate_confusion_matrix(all_labels=train_all_labels.numpy(),
all_predicts=train_all_predicts.numpy(),
classes=classes,
num_classes=num_classes)
# Store training diagnostics.
train_loss = train_running_loss / dataset_sizes["train"]
train_acc = train_running_corrects / dataset_sizes["train"]
if torch.cuda.is_available():
torch.cuda.empty_cache()
# Validation phase.
model.train(mode=False)
val_running_loss = 0.0
val_running_corrects = 0
# Feed forward over all the validation data.
for idx, (val_inputs, val_labels) in enumerate(dataloaders["val"]):
val_inputs = val_inputs.to(device=device)
val_labels = val_labels.to(device=device)
# Feed forward.
with torch.set_grad_enabled(mode=False):
val_outputs = model(val_inputs)
_, val_preds = torch.max(val_outputs, dim=1)
val_loss = criterion(input=val_outputs, target=val_labels)
# Update validation diagnostics.
val_running_loss += val_loss.item() * val_inputs.size(0)
val_running_corrects += torch.sum(val_preds == val_labels.data,
dtype=torch.double)
start = idx * batch_size
end = start + batch_size
val_all_labels[start:end] = val_labels.detach().cpu()
val_all_predicts[start:end] = val_preds.detach().cpu()
calculate_confusion_matrix(all_labels=val_all_labels.numpy(),
all_predicts=val_all_predicts.numpy(),
classes=classes,
num_classes=num_classes)
# Store validation diagnostics.
val_loss = val_running_loss / dataset_sizes["val"]
val_acc = val_running_corrects / dataset_sizes["val"]
if torch.cuda.is_available():
torch.cuda.empty_cache()
scheduler.step()
current_lr = None
for group in optimizer.param_groups:
current_lr = group["lr"]
# Remaining things related to training.
if epoch % save_interval == 0:
epoch_output_path = checkpoints_folder.joinpath(
f"resnet{num_layers}_e{epoch}_va{val_acc:.5f}.pt")
# Confirm the output directory exists.
epoch_output_path.parent.mkdir(parents=True, exist_ok=True)
# Save the model as a state dictionary.
torch.save(obj={
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"epoch": epoch + 1
},
f=str(epoch_output_path))
writer.write(f"{epoch},{train_loss:.4f},"
f"{train_acc:.4f},{val_loss:.4f},{val_acc:.4f}\n")
# Print the diagnostics for each epoch.
print(f"Epoch {epoch} with lr "
f"{current_lr:.15f}: "
f"t_loss: {train_loss:.4f} "
f"t_acc: {train_acc:.4f} "
f"v_loss: {val_loss:.4f} "
f"v_acc: {val_acc:.4f}\n")
# Print training information at the end.
print(f"\ntraining complete in "
f"{(time.time() - since) // 60:.2f} minutes")
def __exit__(self, *args):
torch.set_grad_enabled(self.prev)
return False
def train(self, filename):
best_accuracy = []
for i in range(self.net_count):
self.nets[i] = self.nets[i].to(self.device)
best_accuracy.append(0)
starttime = int(time.time())
input_size = 120
with open(REPLAYS_FOLDER + "/model_training_" + filename +
str(starttime) + ".csv",
'a',
newline='') as csvfile:
headers = ['epoch', 'loss', 'avg_validation_accuracy']
headers.extend(self.dataset_labels)
writer = csv.DictWriter(csvfile, fieldnames=headers, delimiter=',')
writer.writeheader()
for epoch in range(self.max_epochs):
# Reshuffle the indexes in the training batch to ensure the net does not memories the order of data being fed in
np.random.shuffle(self.train_indecis)
train_sampler = SubsetRandomSampler(self.train_indecis)
self.train_loader = torch.utils.data.DataLoader(
self.dataset,
batch_size=self.batch_size,
sampler=train_sampler)
# Training
#self.dataset.set_training(True)
epoch_loss = 0.0
running_loss = []
for j in range(self.net_count):
running_loss.append(0.0)
self.nets[j].train(True)
i = 0
with torch.set_grad_enabled(True):
for local_batch, local_labels in self.train_loader:
# Transfer to GPU
local_batch, local_labels = local_batch.to(
self.device), local_labels.to(self.device)
# Zero the gradients for this batch
i += 1
for j in range(self.net_count):
net = self.nets[j]
optimizer = self.optimizers[j]
optimizer.zero_grad()
# Calculating loss
output = net(local_batch)
loss = self.criterion(output, local_labels)
loss.backward()
# Prevent exploding weights
torch.nn.utils.clip_grad_norm_(net.parameters(), 4)
optimizer.step()
running_loss[j] += loss.item()
epoch_loss += output.shape[0] * loss.item()
if (i % 10 == 0):
correct_in_minibatch = (
local_labels == output.max(
dim=1)[1]).sum()
print(
'[Net: %d, %d, %5d] loss: %.3f acc: %.3f' %
(j + 1, epoch + 1, i + 1,
(running_loss[j] / 10),
correct_in_minibatch.item() /
self.batch_size))
running_loss[j] = 0.0
epoch_loss = epoch_loss / (self.dataset_size *
(1 - self.validation_split))
print('Training loss: {:.4f}'.format(epoch_loss))
print("Validating...")
for j in range(self.net_count):
self.nets[j].train(False)
# Validation
#self.dataset.set_training(False)
epoch_validation_loss = []
correct = []
epoch_loss = []
accuracy = []
for j in range(self.net_count):
epoch_validation_loss.append(0.0)
correct.append(0)
with torch.set_grad_enabled(False):
accuracy_batch = {
'total': {},
'correct': {},
'percent': {}
}
for dataset_label in self.dataset_labels:
accuracy_batch['total'][dataset_label] = 0
accuracy_batch['correct'][dataset_label] = 0
accuracy_batch['percent'][dataset_label] = 0
for local_batch, local_labels in self.validation_loader:
# Transfer to GPU
local_batch, local_labels = local_batch.to(
self.device), local_labels.to(self.device)
# Zero the gradients for this batch
for j in range(self.net_count):
optimizer = self.optimizers[j]
net = self.nets[j]
optimizer.zero_grad()
# Calculating loss
output = net(local_batch)
correct[j] += (local_labels == output.max(
dim=1)[1]).sum().item()
loss = self.criterion(output, local_labels)
epoch_validation_loss[
j] += output.shape[0] * loss.item()
# Calculate the percentages
for index, label in enumerate(local_labels):
local_label_string = self.dataset_labels[label]
accuracy_batch['total'][
local_label_string] += 1
if (output[index].argmax() == label):
accuracy_batch['correct'][
local_label_string] += 1
accuracy_batch['percent'][
local_label_string] = accuracy_batch[
'correct'][
local_label_string] / accuracy_batch[
'total'][local_label_string]
for j in range(self.net_count):
epoch_loss.append(
epoch_validation_loss[j] /
(self.dataset_size * self.validation_split))
accuracy.append(
correct[j] /
(self.dataset_size * self.validation_split))
print('[Net: %d] Validation loss: %.4f accuracy %.3f' %
(j + 1, epoch_loss[j], accuracy[j]))
print(
'[Combined] Sum validation loss: %.4f average accuracy %.3f'
% (np.sum(epoch_loss), np.average(accuracy)))
csv_row = {
'epoch': epoch,
'loss': np.sum(epoch_loss),
'avg_validation_accuracy': np.average(accuracy)
}
for dataset_label in self.dataset_labels:
csv_row[dataset_label] = accuracy_batch['percent'][
dataset_label]
writer.writerow(csv_row)
csvfile.flush()
for j in range(self.net_count):
current_filename = filename + '_' + str(j + 1)
if (accuracy[j] > best_accuracy[j]):
best_accuracy[j] = accuracy[j]
current_filename = filename + '_' + str(j +
1) + '-BEST'
torch.save(
{
'state_dict': self.nets[j].state_dict(),
'input_size': self.input_size,
'labels': self.dataset_labels,
'accuracy': accuracy[j],
'last_row': csv_row,
'loss': epoch_loss[j],
'epoch': epoch
},
os.path.join(CLASSIFIER_FOLDER, current_filename) +
'-weights.pth.tar')
def train_model(image_datasets, arch='vgg19', hidden_units=4096, epochs=25, learning_rate=0.001, gpu=False, checkpoint=''):
# Use command line values when specified
if args.arch:
arch = args.arch
if args.hidden_units:
hidden_units = args.hidden_units
if args.epochs:
epochs = args.epochs
if args.learning_rate:
learning_rate = args.learning_rate
if args.gpu:
gpu = args.gpu
if args.checkpoint:
checkpoint = args.checkpoint
# Using the image datasets, define the dataloaders
dataloaders = {
x: data.DataLoader(image_datasets[x], batch_size=4, shuffle=True, num_workers=2)
for x in list(image_datasets.keys())
}
# Calculate dataset sizes.
dataset_sizes = {
x: len(dataloaders[x].dataset)
for x in list(image_datasets.keys())
}
print('Network architecture:', arch)
print('Number of hidden units:', hidden_units)
print('Number of epochs:', epochs)
print('Learning rate:', learning_rate)
# Load the model
num_labels = len(image_datasets['train'].classes)
model = load_model(arch=arch, num_labels=num_labels, hidden_units=hidden_units)
# Use gpu if selected and available
if gpu and torch.cuda.is_available():
print('Using GPU for training')
device = torch.device("cuda:0")
model.cuda()
else:
print('Using CPU for training')
device = torch.device("cpu")
# Defining criterion, optimizer and scheduler
# Observe that only parameters that require gradients are optimized
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(list(filter(lambda p: p.requires_grad, model.parameters())), lr=learning_rate, momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
for epoch in range(epochs):
print('Epoch {}/{}'.format(epoch + 1, epochs))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'valid']:
if phase == 'train':
scheduler.step()
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(
phase, epoch_loss, epoch_acc))
# deep copy the model
if phase == 'valid' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# Load best model weights
model.load_state_dict(best_model_wts)
# Store class_to_idx into a model property
model.class_to_idx = image_datasets['train'].class_to_idx
# Save checkpoint if requested
if checkpoint:
print ('Saving checkpoint to:', checkpoint)
checkpoint_dict = {
'arch': arch,
'class_to_idx': model.class_to_idx,
'state_dict': model.state_dict(),
'hidden_units': hidden_units
}
torch.save(checkpoint_dict, checkpoint)
# Return the model
return model
def train(train_loader, model, criterion, optimizer, epoch, use_cuda):
# switch to train mode
model.train()
torch.set_grad_enabled(True)
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
end = time.time()
bar = Bar('Processing', max=len(train_loader))
show_step = len(train_loader) // 10
for batch_idx, (inputs, targets) in enumerate(train_loader):
batch_size = inputs.size(0)
if batch_size < args.train_batch:
continue
# measure data loading time
data_time.update(time.time() - end)
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda(async=True)
inputs, targets = torch.autograd.Variable(
inputs), torch.autograd.Variable(targets)
# compute output
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5))
losses.update(loss.data, inputs.size(0))
top1.update(prec1, inputs.size(0))
top5.update(prec5, inputs.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format(
batch=batch_idx + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
if (batch_idx) % show_step == 0:
print(bar.suffix)
bar.next()
bar.finish()
return (losses.avg, top1.avg)
def main(args):
print(args)
CLASSES = args.classes
categories = []
iid = 0
for i in args.classes:
iid += 1
item = {'category': i, 'id': iid, 'name': i, 'supercategory': 'Litter'}
categories.append(item)
if args.dst_img is not None and not os.path.exists(args.dst_img):
os.mkdir(args.dst_img)
num_classes = len(CLASSES)
torch.set_grad_enabled(False)
model = args.checkpoint.split('-')[-1].split('/')[0]
bench = set_model(model, num_classes, args.checkpoint, args.device)
bench.eval()
annotations = {}
annotations['info'] = {
'contributor':
'WiML&DS, Detect Waste in Pomerania',
'date_created':
'12.02.2021',
'description':
'Detect waste is a non-profit, educational, '
'eco project that aims to use Artificial '
'Intelligence for the general good. '
'We have gathered a team of ten carefully selected '
'members and five mentors to work together on the '
'problem of the world’s waste pollution.',
'url':
'detectwaste.ml',
'version':
'v1',
'year':
'2021'
}
annotations['licenses'] = [{
'id':
0,
'name':
'Attribution-NonCommercial 4.0 International',
'url':
'https://creativecommons.org/licenses/by-nc/4.0/legalcode'
}]
annotations['categories'] = categories
annotations['images'] = []
img_id = 0
annotations['annotations'] = []
ann_id = 0
for fname in tqdm(os.listdir(args.src_img)):
img_name = os.path.join(args.src_img, fname)
# read an image and add it to annotations
try:
im = Image.open(img_name).convert('RGB')
w, h = im.size
img_item = {
'file_name': fname,
'height': h,
'id': img_id,
'width': w
}
annotations['images'].append(img_item)
except:
print(f"Error with {img_name}")
continue
# mean-std normalize the input image (batch-size: 1)
img = get_transforms(im)
# propagate through the model
outputs = bench(img.to(args.device))
# keep only predictions above set confidence
bboxes_keep = outputs[0, outputs[0, :, 4] > args.prob_threshold]
probas = bboxes_keep[:, 4:]
# convert boxes to image scales
bboxes_scaled = rescale_bboxes(bboxes_keep[:, :4], im.size,
tuple(img.size()[2:]))
# plot and save demo image
if args.dst_img is not None:
try:
plot_results(im, probas, bboxes_scaled, args.classes,
os.path.join(args.dst_img, 'BBox_' + fname))
except:
print(f"Error with {img_name}")
continue
for p, i in zip(probas, bboxes_scaled):
ann_item = {
'area': (i[2] - i[0]) * (i[3] - i[1]),
'bbox': [i[0], i[1], i[2] - i[0], i[3] - i[1]],
'category_id': int(p[1]),
'id': ann_id,
'image_id': img_id,
'iscrowd': 0,
'probability': format(float(p[0]), '.2f')
}
annotations['annotations'].append(ann_item)
ann_id += 1
img_id += 1
with open(args.dst_coco, 'w') as f:
json.dump(annotations, f)
def trainEnc_MI(stats, model, optimizer, scheduler, dataloaders, device,
kwargs):
"""This funcion performs training of DIM local version without prior distrib. The classifier can be wrapped into the model
using the prior flag, in kwargs dictionary there are othe hyperparameters used for the training. Also tensorbord is available
"""
num_epochs = kwargs['ep']
writer = kwargs['writer']
best_loss = kwargs['best_loss']
t_board = kwargs['t_board']
gamma = kwargs['gamma']
beta = kwargs['beta']
use_prior = kwargs['Prior_Flag']
discriminator = kwargs['discriminator']
gen_epoch = 0
gen_epoch_l = 0
best_model_wts = copy.deepcopy(model.state_dict())
for epoch in range(num_epochs):
stats['disk'].append(check_ext_mem("cl_ext_mem"))
stats['ram'].append(check_ram_usage())
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if use_prior:
discriminator.requires_grad_(False)
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
metrics = defaultdict(float)
epoch_samples = 0
batch_num = 0
for inputs, labels in dataloaders[phase]:
labels = labels.type(torch.long)
torch.cuda.empty_cache()
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
#print(inputs.size(),inputs.dtype,labels.size(),labels.dtype)
# forward
with torch.set_grad_enabled(phase == 'train'):
E_phi, C_phi, A_phi = model(inputs)
loss = 0
#we use infoNCE MI approx. since it is more stable eventually for other exp call compute dim loss
#fenchel_dual_loss(C_phi, E_phi, measure='JSD')
#function already implemented
loss_MI = infonce_loss(C_phi, E_phi)
if use_prior:
loss += beta * loss_MI
else:
loss = loss_MI
metrics['MI_loss'] += loss.data.cpu().numpy()
################################## section for prior Loss
if use_prior:
Q_samples = discriminator(A_phi)
disc_loss = compute_loss(Q_samples, labels, metrics)
loss += gamma * disc_loss
metrics['loss'] += loss.data.cpu().numpy()
################################## backward and tensorboard: othre quantities can be added
if phase == 'train':
loss.backward()
optimizer.step()
if batch_num % 20 == 0 and t_board:
# ...log the running loss
writer.add_scalar('training loss', loss,
epoch * gen_epoch + batch_num)
if phase == 'val' and t_board:
if batch_num % 10 == 0:
# ...log the running loss
writer.add_scalar('Validation loss', loss,
epoch * gen_epoch_l + batch_num)
# statistics
batch_num += 1
############ end of epochs
epoch_loss = metrics['loss'] / batch_num
if epoch == 0 and phase == 'train':
gen_epoch = batch_num
if epoch == 0 and phase == 'val':
gen_epoch_l = batch_num
if phase == 'train':
avg = metrics['loss'] / batch_num
scheduler.step() #avg
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
#### Write and Save
print_metrics(metrics, batch_num, phase)
if phase == 'val' and epoch_loss < best_loss:
print("saving best model")
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
############## Time Info
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
del best_model_wts
return model, stats
def setUp(self):
torch.set_grad_enabled(False)
def trainEnc_MIadv(stats, model, optimizer, scheduler, dataloaders, device,
kwargs):
"""This funcion performs training of DIM local version without prior distrib. It can be wrapped into the DNN
module as a class function
"""
num_epochs = kwargs['ep']
writer = kwargs['writer']
best_loss = kwargs['best_loss']
t_board = kwargs['t_board']
gamma = kwargs['gamma']
beta = kwargs['beta']
use_prior = kwargs['Prior_Flag']
gen_epoch = 0
gen_epoch_l = 0
best_model_wts = copy.deepcopy(model.state_dict())
model2 = copy.deepcopy(model)
model2.load_state_dict(model.state_dict())
for epoch in range(num_epochs):
stats['disk'].append(check_ext_mem("cl_ext_mem"))
stats['ram'].append(check_ram_usage())
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
metrics = defaultdict(float)
epoch_samples = 0
batch_num = 0
for inputs, labels in dataloaders[phase]:
if len(inputs.shape) == 5:
inputs_ref = inputs[:, :, :, :, 1].to(device)
inputs = inputs[:, :, :, :, 0].to(device)
else:
inputs = inputs.to(device)
labels = labels.type(torch.long)
torch.cuda.empty_cache()
labels = labels.to(device)
model2.requires_grad_(False)
optimizer.zero_grad()
#print(inputs.size(),inputs_ref.size())
# forward
with torch.set_grad_enabled(phase == 'train'):
loss = 0
#we use NCE MI approx. since it is more stable eventually for other exp call compute dim loss
#function already implemented i.e. fenchel_dual_loss(C_phi, E_phi, measure='JSD')
if use_prior:
### combination of 3 losses: MI same batch MI across batches and classification score
###
_, C_phi_ref, _, _ = model2(inputs_ref)
E_phi, C_phi, A_phi, pred = model(inputs)
loss_MI = infonce_loss(C_phi, E_phi)
metrics['loss_MI'] += loss_MI.data.cpu().numpy()
loss += beta * loss_MI
loss_MI_ref = infonce_loss(C_phi_ref.detach(), E_phi)
metrics['MI_REF_loss'] += loss_MI_ref.data.cpu().numpy(
)
loss += beta * loss_MI_ref
losscl = compute_loss(pred, labels, metrics)
metrics['CL_loss'] += losscl.data.cpu().numpy()
loss += beta * losscl
else:
### combination of 2 losses: MI info and classification
E_phi, C_phi, A_phi, pred = model(inputs)
loss_MI = infonce_loss(C_phi, E_phi)
metrics['loss_MI'] += loss_MI.data.cpu().numpy()
losscl = compute_loss(pred, labels, metrics)
metrics['CL_loss'] += losscl.data.cpu().numpy()
loss = beta * loss_MI + beta * losscl
################################## section for prior Loss
metrics['loss'] += loss.data.cpu().numpy()
################################## backward and tensorboard: othre quantities can be added
if phase == 'train':
loss.backward()
optimizer.step()
if batch_num % 20 == 0 and t_board:
# ...log the running loss
writer.add_scalar('training loss', loss,
epoch * gen_epoch + batch_num)
if phase == 'val' and t_board:
if batch_num % 10 == 0:
# ...log the running loss
writer.add_scalar('Validation loss', loss,
epoch * gen_epoch_l + batch_num)
# statistics
batch_num += 1
############ end of epochs
epoch_loss = metrics['loss'] / batch_num
if epoch == 0 and phase == 'train':
gen_epoch = batch_num
if epoch == 0 and phase == 'val':
gen_epoch_l = batch_num
if phase == 'train':
avg = metrics['loss'] / batch_num
scheduler.step() #avg
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
#### Write and Save
print_metrics(metrics, batch_num, phase)
if phase == 'val' and epoch_loss < best_loss:
print("saving best model")
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
model2.load_state_dict(model.state_dict())
############## Time Info
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
del best_model_wts, model2
return model, stats
def forward(self, data, im_info, roidb=None, **rpn_kwargs):
if cfg.PYTORCH_VERSION_LESS_THAN_040:
return self._forward(data, im_info, roidb, **rpn_kwargs)
else:
with torch.set_grad_enabled(self.training):
return self._forward(data, im_info, roidb, **rpn_kwargs)
def train_classifier(stats, model, optimizer, scheduler, dataloaders, device,
kwargs):
"""This funcion performs training of classifier HEAD. In kwargs dictionary there are othe hyperparameters used for the training.
Also tensorbord is available
"""
num_epochs = kwargs['ep']
writer = kwargs['writer']
best_loss = kwargs['best_loss']
t_board = kwargs['t_board']
gen_epoch = 0
gen_epoch_l = 0
best_model_wts = copy.deepcopy(model.state_dict())
for epoch in range(num_epochs):
stats['disk'].append(check_ext_mem("cl_ext_mem"))
stats['ram'].append(check_ram_usage())
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
metrics = defaultdict(float)
epoch_samples = 0
batch_num = 0
for inputs, labels in dataloaders[phase]:
labels = labels.type(torch.long)
torch.cuda.empty_cache()
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
# forward
# track history if only in train
#print(inputs.size(),inputs.dtype,labels.size(),labels.dtype)
with torch.set_grad_enabled(phase == 'train'):
out = model(inputs)
loss = 0
loss = compute_loss(out, labels, metrics)
if phase == 'train':
loss.backward()
optimizer.step()
if batch_num % 20 == 0 and t_board:
writer.add_scalar('training loss', loss,
epoch * gen_epoch + batch_num)
if phase == 'val' and t_board:
if batch_num % 10 == 0:
writer.add_scalar('Validation loss', loss,
epoch * gen_epoch_l + batch_num)
# statistics
batch_num += 1
epoch_loss = metrics['loss'] / batch_num
if epoch == 0 and phase == 'train':
gen_epoch = batch_num
if epoch == 0 and phase == 'val':
gen_epoch_l = batch_num
if phase == 'train':
scheduler.step() #avg
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
print_metrics(metrics, batch_num, phase)
if phase == 'val' and epoch_loss < best_loss:
print("saving best model")
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
del best_model_wts
return model, stats
def train(train_loader, model, optimizer, criterion, writer, count, epoch):
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to train mode
phase = 'train'
model.train()
iter_len = train_loader.__len__()
for i in range(0, iter_len // train_cfg['per_batch_size']):
### Adjust learning rate
# lr = adjust_lr_test(optimizer, count, train_cfg['init_lr'])
model.train()
batch_iterator = iter(
DataLoader(train_loader,
train_cfg['per_batch_size'],
shuffle=True,
num_workers=args.num_workers,
collate_fn=detection_collate))
input, target = next(batch_iterator)
if train_cfg['cuda']:
target = target.cuda()
input = input.cuda()
# input_var = torch.autograd.Variable(input)
# target_var = torch.autograd.Variable(target)
# compute output
with torch.set_grad_enabled(phase == 'train'):
output = model(input)
loss = criterion(output, target.long())
acc1, acc5 = accu(output.data, target, topk=(1, 5))
# measure accuracy and record loss
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
count += 1
writer.add_scalar('lr',
optimizer.param_groups[0]['lr'],
global_step=count)
writer = add_summery(writer, 'train', loss.data, acc1, acc5, count)
if i % train_cfg['print_que'] == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'lr: [{3}]\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch,
i,
len(train_loader) // train_cfg['per_batch_size'],
optimizer.param_groups[0]['lr'],
loss=losses,
top1=top1,
top5=top5))
return count, losses.avg
def train_model(model,
criterion,
optimizer,
scheduler,
fn_save,
num_epochs=25):
since = time.time()
#best_model_wts = copy.deepcopy(model.state_dict())
least_loss = 10000.0
best_ep = 0
tv_hist = {'train': [], 'val': []}
for epoch in range(num_epochs):
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
# Iterate over data.
for inputs, input_patch, labels, agatston_scores, manual_scores, img_id in dataloaders[
phase]:
# Get images from inputs
#print('*'*10 + ' printing inputs and labels ' + '*'*10)
manual_scores = manual_scores.type(torch.FloatTensor)
agatston_scores = agatston_scores.type(torch.FloatTensor)
inputs = inputs.to(device)
input_patch = input_patch.to(device)
labels = labels.to(device)
manual_scores = manual_scores.to(device)
agatston_scores = agatston_scores.to(device)
list = [inputs, manual_scores, agatston_scores, labels]
# print(img_id)
# print(manual_scores)
# print(agatston_scores)
# print(labels)
# time.sleep(10)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
myresnet.myinputs = list
# print('{}'.format(list.__sizeof__()))
# outputs, mu, layvar = model(inputs)
outputs, code = model(inputs)
inputs = inputs.type(torch.cuda.FloatTensor)
inputs_array = inputs.data.cpu().numpy()
outputs_array = outputs.data.cpu().numpy()
save_sample(inputs_array[0, :, :, :], 'inputs')
save_sample(outputs_array[0, :, :, :], 'outputs')
# inputs = inputs.type(torch.cuda.LongTensor)
# outputs = outputs.type(torch.cuda.LongTensor)
# print('input size {}'.format(inputs.shape))
# print('input type {}'.format(inputs.dtype))
# print('output size {}'.format(outputs.shape))
# print('output type {}'.format(outputs.dtype))
loss = criterion(outputs, inputs)
# print('loss shape {}'.format(loss.shape))
# print('loss is {}'.format(loss))
# time.sleep(1)
# time.sleep(30)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
# running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / dataset_sizes[phase]
# epoch_acc = running_corrects.double() / dataset_sizes[phase]
tv_hist[phase].append([epoch_loss])
# deep copy the model
if phase == 'val' and epoch_loss <= least_loss:
least_loss = epoch_loss
best_ep = epoch
#best_model_wts = copy.deepcopy(model.state_dict())
torch.save(model.state_dict(), fn_save)
print('ep {}/{} - Train loss: {:.4f}, Val loss: {:.4f}'.format(
epoch + 1, num_epochs, tv_hist['train'][-1][0],
tv_hist['val'][-1][0]))
training_progress[epoch][0] = tv_hist['train'][-1][0]
# training_progress[epoch][1] = tv_hist['train'][-1][1]
training_progress[epoch][1] = tv_hist['val'][-1][0]
# training_progress[epoch][3] = tv_hist['val'][-1][1]
#print('-' * 10)
#print('{} Loss: {:.4f} Acc: {:.4f}'.format(
# phase, epoch_loss, epoch_acc))
time_elapsed = time.time() - since
print('*' * 10 + 'Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('*' * 10 +
'Least Val Loss: {:4f} at epoch {}'.format(least_loss, best_ep))
print()
# load best model weights
#model.load_state_dict(best_model_wts)
#return model
return tv_hist
