def _train(args, fine=False):
logger = tk.log.get(__name__)
logger.info(f'args: {args}')
split_seed = int(MODEL_NAME.encode('utf-8').hex(), 16) % 10000000
MODELS_DIR.mkdir(parents=True, exist_ok=True)
(MODELS_DIR / 'split_seed.txt').write_text(str(split_seed))
X, y = _data.load_train_data()
ti, vi = tk.ml.cv_indices(X, y, cv_count=CV_COUNT, cv_index=args.cv_index, split_seed=split_seed, stratify=False)
(X_train, y_train), (X_val, y_val) = (X[ti], y[ti]), (X[vi], y[vi])
logger.info(f'cv_index={args.cv_index}: train={len(y_train)} val={len(y_val)}')
network, lr_multipliers = _create_network()
gen = tk.generator.Generator()
if fine:
pseudo_size = len(y_train) // 2
X_train = np.array(list(X_train) + [None] * pseudo_size)
y_train = np.array(list(y_train) + [None] * pseudo_size)
X_test = _data.load_test_data()
_, pi = tk.ml.cv_indices(X_test, np.zeros((len(X_test),)), cv_count=CV_COUNT, cv_index=args.cv_index, split_seed=split_seed, stratify=False)
#pred_test = predict_all('test', None, use_cache=True)[(args.cv_index + 1) % CV_COUNT] # cross-pseudo-labeling
import stack_res
pred_test = stack_res.predict_all('test', None, use_cache=True)[(args.cv_index + 1) % CV_COUNT] # cross-pseudo-labeling
gen.add(tk.generator.RandomPickData(X_test[pi], pred_test[pi]))
gen.add(tk.image.RandomFlipLR(probability=0.5, with_output=True))
gen.add(tk.image.Padding(probability=1, with_output=True))
gen.add(tk.image.RandomRotate(probability=0.25, with_output=True))
gen.add(tk.image.RandomCrop(probability=1, with_output=True))
gen.add(tk.image.RandomAugmentors([
tk.image.RandomBlur(probability=0.125),
tk.image.RandomUnsharpMask(probability=0.125),
tk.image.RandomBrightness(probability=0.25),
tk.image.RandomContrast(probability=0.25),
], probability=0.125))
gen.add(tk.image.Resize((101, 101), with_output=True))
model = tk.dl.models.Model(network, gen, batch_size=BATCH_SIZE)
if fine:
model.load_weights(MODELS_DIR / f'model.fold{args.cv_index}.h5')
model.compile(sgd_lr=0.001 / 128 if fine else 0.1 / 128, loss=tk.dl.losses.lovasz_hinge_elup1,
metrics=[tk.dl.metrics.binary_accuracy], lr_multipliers=lr_multipliers, clipnorm=10.0)
model.fit(
X_train, y_train, validation_data=(X_val, y_val),
epochs=EPOCHS // 3 if fine else EPOCHS,
cosine_annealing=True, mixup=False)
model.save(MODELS_DIR / f'model.fold{args.cv_index}.h5', include_optimizer=False)
if tk.dl.hvd.is_master():
_evaluation.log_evaluation(y_val, model.predict(X_val))
def predict_all(data_name, X):
import stack_dense
import stack_drop
import stack_res
def _get(pred):
return pred if data_name == 'val' else np.mean(pred, axis=0)
return np.average([
_get(stack_dense.predict_all(data_name, X, use_cache=True)),
_get(stack_drop.predict_all(data_name, X, use_cache=True)),
_get(stack_res.predict_all(data_name, X, use_cache=True)),
],
weights=[1, 1.5, 1.3],
axis=0)
