def predict_sep(model, valid_loader, criterion, device, classification=False):
model.eval()
test_loss = 0.0
score_all1 = []
score_all2 = []
score_all3 = []
score_all4 = []
with torch.no_grad():
for step, (features, targets) in enumerate(tqdm(valid_loader)):
features, targets = features.to(device), targets.to(device)
if classification:
logits, _ = model(features)
else:
logits = model(features)
loss = criterion(logits, targets)
targets = targets.float().cpu().numpy().astype("int8")
logits = torch.sigmoid(logits.view(
targets.shape)).float().cpu().numpy().astype("float16")
test_loss += loss.item()
for i in range(4):
th, score, ths, scores = search_threshold(
targets[:, i, :, :], logits[:, i, :, :])
if i == 0:
score_all1.append(scores)
if i == 1:
score_all2.append(scores)
if i == 2:
score_all3.append(scores)
if i == 3:
score_all4.append(scores)
del features, targets, logits
gc.collect()
return test_loss / (step + 1), np.array(score_all1), np.array(
score_all2), np.array(score_all3), np.array(score_all4), ths
def main():
with timer('load data'):
df = pd.read_csv(FOLD_PATH)
with timer('preprocessing'):
train_df, val_df = df[df.fold_id != FOLD_ID], df[df.fold_id == FOLD_ID]
train_augmentation = Compose([
Flip(p=0.5),
OneOf(
[
#ElasticTransform(p=0.5, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion(p=0.5),
OpticalDistortion(p=0.5, distort_limit=2, shift_limit=0.5)
],
p=0.5),
#OneOf([
# ShiftScaleRotate(p=0.5),
## RandomRotate90(p=0.5),
# Rotate(p=0.5)
#], p=0.5),
OneOf([
Blur(blur_limit=8, p=0.5),
MotionBlur(blur_limit=8, p=0.5),
MedianBlur(blur_limit=8, p=0.5),
GaussianBlur(blur_limit=8, p=0.5)
],
p=0.5),
OneOf(
[
#CLAHE(clip_limit=4, tile_grid_size=(4, 4), p=0.5),
RandomGamma(gamma_limit=(100, 140), p=0.5),
RandomBrightnessContrast(p=0.5),
RandomBrightness(p=0.5),
RandomContrast(p=0.5)
],
p=0.5),
OneOf([
GaussNoise(p=0.5),
Cutout(num_holes=10, max_h_size=10, max_w_size=20, p=0.5)
],
p=0.5)
])
val_augmentation = None
train_dataset = SeverDataset(train_df,
IMG_DIR,
IMG_SIZE,
N_CLASSES,
id_colname=ID_COLUMNS,
transforms=train_augmentation)
val_dataset = SeverDataset(val_df,
IMG_DIR,
IMG_SIZE,
N_CLASSES,
id_colname=ID_COLUMNS,
transforms=val_augmentation)
train_loader = DataLoader(train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2)
val_loader = DataLoader(val_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=2)
del train_df, val_df, df, train_dataset, val_dataset
gc.collect()
with timer('create model'):
model = smp.UnetPP('se_resnext101_32x4d',
encoder_weights='imagenet',
classes=N_CLASSES,
encoder_se_module=True,
decoder_semodule=True,
h_columns=False,
deep_supervision=True)
model.load_state_dict(torch.load(model_path))
model.to(device)
criterion = FocalLovaszLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=3e-4)
scheduler = CosineAnnealingLR(optimizer, T_max=CLR_CYCLE, eta_min=3e-5)
#scheduler = GradualWarmupScheduler(optimizer, multiplier=1.1, total_epoch=CLR_CYCLE*2, after_scheduler=scheduler_cosine)
model, optimizer = amp.initialize(model,
optimizer,
opt_level="O1",
verbosity=0)
with timer('train'):
train_losses = []
valid_losses = []
best_model_loss = 999
best_model_ep = 0
checkpoint = 0
for epoch in range(1, EPOCHS + 1):
if epoch % (CLR_CYCLE * 2) == 0:
if epoch != 0:
y_val = y_val.reshape(-1, N_CLASSES, IMG_SIZE[0],
IMG_SIZE[1])
best_pred = best_pred.reshape(-1, N_CLASSES, IMG_SIZE[0],
IMG_SIZE[1])
for i in range(N_CLASSES):
th, score, _, _ = search_threshold(
y_val[:, i, :, :], best_pred[:, i, :, :])
LOGGER.info(
'Best loss: {} Best Dice: {} on epoch {} th {} class {}'
.format(round(best_model_loss, 5), round(score, 5),
best_model_ep, th, i))
checkpoint += 1
best_model_loss = 999
LOGGER.info("Starting {} epoch...".format(epoch))
tr_loss = train_one_epoch_dsv(model, train_loader, criterion,
optimizer, device)
train_losses.append(tr_loss)
LOGGER.info('Mean train loss: {}'.format(round(tr_loss, 5)))
valid_loss, val_pred, y_val = validate_dsv(model, val_loader,
criterion, device)
valid_losses.append(valid_loss)
LOGGER.info('Mean valid loss: {}'.format(round(valid_loss, 5)))
scheduler.step()
if valid_loss < best_model_loss:
torch.save(
model.state_dict(),
'{}_fold{}_ckpt{}.pth'.format(EXP_ID, FOLD_ID, checkpoint))
best_model_loss = valid_loss
best_model_ep = epoch
best_pred = val_pred
del val_pred
gc.collect()
with timer('eval'):
y_val = y_val.reshape(-1, N_CLASSES, IMG_SIZE[0], IMG_SIZE[1])
best_pred = best_pred.reshape(-1, N_CLASSES, IMG_SIZE[0], IMG_SIZE[1])
for i in range(N_CLASSES):
th, score, _, _ = search_threshold(y_val[:, i, :, :],
best_pred[:, i, :, :])
LOGGER.info(
'Best loss: {} Best Dice: {} on epoch {} th {} class {}'.
format(round(best_model_loss, 5), round(score, 5),
best_model_ep, th, i))
xs = list(range(1, len(train_losses) + 1))
plt.plot(xs, train_losses, label='Train loss')
plt.plot(xs, valid_losses, label='Val loss')
plt.legend()
plt.xticks(xs)
plt.xlabel('Epochs')
plt.savefig("loss.png")
for i, (train_index, val_index) in enumerate(kfold):
X, y, val_X, val_y = X_train[train_index], y_train[
train_index], X_train[val_index], y_train[val_index]
model = models.getModel(param, name)
filepath = param['subject_ckp_path'] + name + '-' + str(i + 1)
if not os.path.exists(filepath):
model.fit(X, y)
joblib.dump(model, filepath)
model = joblib.load(filepath)
y_pred = model.predict_proba(val_X)
y_target[val_index, n] = y_pred
y_test_pred[:, n * fold + i] = model.predict_proba(X_test)
# threshold = util.search_threshold(np.squeeze(val_y), np.squeeze(y_pred))
# pre_label = util.get_label_pre(y_pred, threshold)
# val_label = util.get_label_real(val_y)
# f1 = util.compute_f1_subject(val_label, pre_label)
# print('f1 score:%.6f' %f1)
n += 1
y_target_pred = y_target.mean(axis=1)
threshold = util.search_threshold(y_train, y_target_pred)
pre_label = util.get_label_pre(y_target_pred, threshold)
val_label = util.get_label_real(y_train)
f1 = util.compute_f1_subject(val_label, pre_label)
print('Final threshold:', threshold)
print('Final f1 score:%.6f' % f1)
test_logits = np.squeeze(y_test_pred.mean(axis=1))
test_label = util.get_label_pre(test_logits, threshold)
util.output_subject(test_id, test_content, test_label, index2label,
param['output_path'])
end = min((i + 1) * batch_size, text_num)
batch_shuffle = np.random.permutation(np.arange(end - start))
batch_labels = train_label[start:end][batch_shuffle]
batch_input_idsList = train_idsList[start:end][batch_shuffle]
batch_input_masksList = train_masksList[start:end][batch_shuffle]
batch_segment_idsList = train_segment[start:end][batch_shuffle]
l, _ = sess.run([loss, train_op], feed_dict = {input_ids:batch_input_idsList, input_mask:batch_input_masksList, segment_ids:batch_segment_idsList, input_labels:batch_labels, learning_rate:lr, train_flag:True, dropout_keep_prob:0.9})
train_loss += l
if i % 50 == 0 and i > 0:
val_loss = sess.run(loss, feed_dict = {input_ids:val_idsList, input_mask:val_masksList, segment_ids:val_segment_idsList, input_labels:val_y, dropout_keep_prob:1.0})
print('iter_num = {}\ttrain loss = {:.5f}\tval_loss = {:.5f}'.format(i, train_loss/i, val_loss))
y_pred = sess.run(log_probs, feed_dict = {input_ids:val_idsList, input_mask:val_masksList, segment_ids:val_segment_idsList, dropout_keep_prob:1.0})
y_target[val_index] = y_pred
test_pred = sess.run(log_probs, feed_dict = {input_ids:ids_test, input_mask:masks_test, segment_ids:segment_ids_test, dropout_keep_prob:1.0})
y_test_pred += test_pred / fold
threshold = util.search_threshold(np.squeeze(val_y), np.squeeze(y_pred))
pre_label = util.get_label_pre(y_pred, threshold)
val_label = util.get_label_real(val_y)
f1 = util.compute_f1_subject(val_label, pre_label)
print('Epoch\t%d/%d\tf1 score:%.6f:' %(epoch+1, epochs, f1))
y_target_pred = y_target.mean(axis = 1)
threshold = util.search_threshold(y_train, y_target_pred)
pre_label = util.get_label_pre(y_target_pred, threshold)
val_label = util.get_label_real(y_train)
f1 = util.compute_f1_subject(val_label, pre_label)
print('Final threshold:', threshold)
print('Final f1 score:%.6f' %f1)
test_logits = np.squeeze(y_test_pred)
test_label = util.get_label_pre(test_logits, threshold)
def main(seed):
with timer('load data'):
df = pd.read_csv(FOLD_PATH)
with timer('preprocessing'):
val_df = df[df.fold_id == FOLD_ID]
val_augmentation = None
val_dataset = SeverDataset(val_df,
IMG_DIR,
IMG_SIZE,
N_CLASSES,
id_colname=ID_COLUMNS,
transforms=val_augmentation)
val_loader = DataLoader(val_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=8)
del val_df, df, val_dataset
gc.collect()
with timer('create model'):
models = []
for p in base_model_res:
model = smp.Unet('resnet34',
encoder_weights="imagenet",
classes=N_CLASSES,
encoder_se_module=True,
decoder_semodule=True,
h_columns=False,
skip=True,
act="swish",
freeze_bn=True,
classification=CLASSIFICATION,
attention_type="cbam",
center=True)
model.load_state_dict(torch.load(p))
model.to(device)
model.eval()
models.append(model)
model = smp_old.Unet('resnet34',
encoder_weights="imagenet",
classes=N_CLASSES,
encoder_se_module=True,
decoder_semodule=True,
h_columns=False,
skip=True,
act="swish",
freeze_bn=True,
classification=CLASSIFICATION)
model.load_state_dict(torch.load(base_model_res_old))
model.to(device)
model.eval()
models.append(model)
criterion = torch.nn.BCEWithLogitsLoss()
with timer('predict'):
valid_loss, y_pred, y_true, cls = predict_ensemble(
models,
val_loader,
criterion,
device,
classification=CLASSIFICATION)
LOGGER.info('Mean valid loss: {}'.format(round(valid_loss, 5)))
scores = []
for i, th in enumerate(ths):
if i == 0:
continue
sum_val_preds = np.sum(
y_pred[:, i, :, :].reshape(len(y_pred), -1) > th, axis=1)
best = 0
for n_th, remove_mask_pixel in enumerate(
[200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]):
val_preds_ = copy.deepcopy(y_pred[:, i, :, :])
val_preds_[sum_val_preds < remove_mask_pixel] = 0
threshold_after_remove, score, _, _ = search_threshold(
y_true[:, i, :, :], val_preds_)
LOGGER.info('dice={} on th={} on {}'.format(
score, threshold_after_remove, remove_mask_pixel))
if score >= best:
best = score
else:
break
scores.append(score)
LOGGER.info('holdout dice={}'.format(np.mean(scores)))