def __init__(self, model, train_loader, valid_loader, config):
self.config = config
self.train_loader = train_loader
self.valid_loader = valid_loader
self.model = model.to(self.config.device)
self.logger = init_logger(self.config.log_dir, 'train_main.log')
self.tb_logger = init_tb_logger(self.config.log_dir, 'train_main')
self.log('\n'.join(
[f"{k} = {v}" for k, v in self.config.__dict__.items()]))
self.summary_loss = AverageMeter()
self.evaluator = Evaluator()
self.criterion = torch.nn.CrossEntropyLoss(
ignore_index=self.config.ignore_index)
self.u_criterion = torch.nn.CrossEntropyLoss(
ignore_index=self.config.ignore_index)
train_params = [{
'params': getattr(model, 'encoder').parameters(),
'lr': self.config.lr
}, {
'params': getattr(model, 'decoder').parameters(),
'lr': self.config.lr * 10
}]
self.optimizer = RAdam(train_params,
weight_decay=self.config.weight_decay)
self.scheduler = CosineAnnealingWarmRestarts(self.optimizer,
T_0=2,
T_mult=2,
eta_min=1e-6)
self.n_ensemble = 0
self.epoch = 0
self.best_epoch = 0
self.best_loss = np.inf
self.best_score = -np.inf
def __init__(self, model, train_loader, valid_loader, fold, config, seed):
self.config = config
self.seed = seed
self.device = self.config.device
self.train_loader = train_loader
self.valid_loader = valid_loader
self.model = model.to(self.device)
self.fold = fold
self.logger = init_logger(
config.log_dir, f'train_seed{self.seed}_fold{self.fold}.log')
self.tb_logger = init_tb_logger(
config.log_dir, f'train_seed{self.seed}_fold{self.fold}')
if self.fold == 0:
self.log('\n'.join(
[f"{k} = {v}" for k, v in self.config.__dict__.items()]))
self.criterion = SmoothBCEwLogits(smoothing=self.config.smoothing)
self.evaluator = nn.BCEWithLogitsLoss()
self.summary_loss = AverageMeter()
self.history = {'train': [], 'valid': []}
self.optimizer = Adam(self.model.parameters(),
lr=config.lr,
weight_decay=self.config.weight_decay)
self.scheduler = OneCycleLR(optimizer=self.optimizer,
pct_start=0.1,
div_factor=1e3,
max_lr=1e-2,
epochs=config.n_epochs,
steps_per_epoch=len(train_loader))
self.scaler = GradScaler() if config.fp16 else None
self.epoch = 0
self.best_epoch = 0
self.best_loss = np.inf
def train_fold(train_config, distrib_config, pipeline_name, log_dir, fold_id,
train_dataloader, valid_dataloader, evaluator1, evaluator2):
if distrib_config['LOCAL_RANK'] == 0:
fold_logger = init_logger(log_dir, 'train_fold_{}.log'.format(fold_id))
fold_tb_logger = init_tb_logger(log_dir,
'train_fold_{}'.format(fold_id))
best_checkpoint_folder = Path(log_dir,
train_config['CHECKPOINTS']['BEST_FOLDER'])
best_checkpoint_folder.mkdir(exist_ok=True, parents=True)
checkpoints_history_folder = Path(
log_dir, train_config['CHECKPOINTS']['FULL_FOLDER'],
'fold{}'.format(fold_id))
checkpoints_history_folder.mkdir(exist_ok=True, parents=True)
checkpoints_topk = train_config['CHECKPOINTS']['TOPK']
calculation_name = '{}_fold{}'.format(pipeline_name, fold_id)
device1 = train_config['DEVICE'] + ':0'
device2 = train_config['DEVICE'] + ':1'
module = importlib.import_module(train_config['MODEL1']['PY'])
model_function = getattr(module, train_config['MODEL1']['CLASS'])
model1 = model_function(**train_config['MODEL1']['ARGS'])
module = importlib.import_module(train_config['MODEL2']['PY'])
model_function = getattr(module, train_config['MODEL2']['CLASS'])
model2 = model_function(**train_config['MODEL2']['ARGS'])
pretrained_model1_path = Path(train_config['MODEL1']['pretrain'])
pretrained_model2_path = Path(train_config['MODEL2']['pretrain'])
if pretrained_model1_path.is_file():
state_dict1 = torch.load(pretrained_model1_path,
map_location=lambda storage, loc: storage)
model1.load_state_dict(state_dict1)
state_dict2 = torch.load(pretrained_model2_path,
map_location=lambda storage, loc: storage)
model2.load_state_dict(state_dict2)
if distrib_config['LOCAL_RANK'] == 0:
fold_logger.info(
'load model from {}'.format(pretrained_model1_path))
fold_logger.info(
'load model from {}'.format(pretrained_model2_path))
loss_args = train_config['CRITERION']
loss_fn = SegmentationLosses(
weight=loss_args['weight'],
size_average=loss_args['size_average'],
batch_average=loss_args['batch_average'],
ignore_index=loss_args['ignore_index'],
cuda=loss_args['cuda']).build_loss(mode=loss_args['mode'])
if train_config['OPTIMIZER']['CLASS'] == 'RAdam':
optimizer_class = getattr(radam, train_config['OPTIMIZER']['CLASS'])
else:
optimizer_class = getattr(torch.optim,
train_config['OPTIMIZER']['CLASS'])
train_params1 = [{
'params': model1.get_1x_lr_params(),
'lr': train_config['OPTIMIZER']['ARGS']['lr']
}, {
'params': model1.get_10x_lr_params(),
'lr': train_config['OPTIMIZER']['ARGS']['lr'] * 10
}]
optimizer1 = optimizer_class(train_params1,
**train_config['OPTIMIZER']['ARGS'])
train_params2 = [{
'params': model2.get_1x_lr_params(),
'lr': train_config['OPTIMIZER']['ARGS']['lr']
}, {
'params': model2.get_10x_lr_params(),
'lr': train_config['OPTIMIZER']['ARGS']['lr'] * 10
}]
optimizer2 = optimizer_class(train_params2,
**train_config['OPTIMIZER']['ARGS'])
scheduler_class = getattr(torch.optim.lr_scheduler,
train_config['SCHEDULER']['CLASS'])
scheduler1 = scheduler_class(optimizer1,
**train_config['SCHEDULER']['ARGS'])
scheduler2 = scheduler_class(optimizer2,
**train_config['SCHEDULER']['ARGS'])
n_epoches = train_config['EPOCHS']
accumulation_step = train_config['ACCUMULATION_STEP']
early_stopping = train_config['EARLY_STOPPING']
if distrib_config['LOCAL_RANK'] != 0:
fold_logger = None
fold_tb_logger = None
best_epoch, best_score = Learning(
distrib_config, optimizer1, optimizer2, loss_fn, evaluator1,
evaluator2, device1, device2, n_epoches, scheduler1, scheduler2,
accumulation_step, early_stopping, fold_logger, fold_tb_logger,
best_checkpoint_folder, checkpoints_history_folder, checkpoints_topk,
calculation_name).run_train(model1, model2, train_dataloader,
valid_dataloader)
fold_logger.info(f'Best Epoch : {best_epoch}, Best Score : {best_score}')
fold_logger.info(f'Best Epoch : {best_epoch}, Best Score : {best_score}')
if __name__ == '__main__':
args = argparser()
train_config = load_yaml(args.config_path)
distrib_config = {}
distrib_config['LOCAL_RANK'] = args.local_rank
root_dir = Path(train_config['DIRECTORY']['ROOT_DIRECTORY'])
data_dir = Path(train_config['DIRECTORY']['DATA_DIRECTORY'])
log_dir = root_dir / train_config['DIRECTORY']['LOGGER_DIRECTORY']
log_dir.mkdir(exist_ok=True, parents=True)
if distrib_config['LOCAL_RANK'] == 0:
main_logger = init_logger(log_dir, 'train_main.log')
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.enabled = True
torch.backends.cudnn.deterministic = True
SEED = train_config['SEED']
seed_everything(SEED)
if distrib_config['LOCAL_RANK'] == 0:
main_logger.info(train_config)
# if "DEVICE_LIST" in train_config:
# os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(map(str, train_config["DEVICE_LIST"]))
# if len(train_config['DEVICE_LIST']) > 1:
# distrib_config['DISTRIBUTED'] = True
# torch.cuda.set_device(distrib_config['LOCAL_RANK'])
logit, _ = model(feature)
pred = torch.sigmoid(logit)
preds.append(pred.detach().cpu().numpy())
preds = np.concatenate(preds, axis=0)
return preds
# ref: https://www.kaggle.com/hiramcho/moa-tabnet-with-pca-rank-gauss?select=gauss_rank_scaler.py
if __name__ == '__main__':
# Backup Code
dst_dir = os.path.join(config.log_dir, 'code')
if not os.path.isdir(dst_dir):
copytree(config.bkup_dir, dst_dir, ignore=ignore_patterns('.*'))
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(map(str, config.device))
main_logger = init_logger(config.log_dir, f'train_main.log')
main_logger.info('\n'.join([f"{k} = {v}" for k, v in config.__dict__.items()]))
# Load Data
train_features = pd.read_csv(f'{config.data_dir}/train_features.csv')
train_targets_scored = pd.read_csv(f'{config.data_dir}/train_targets_scored.csv')
train_targets_nonscored = pd.read_csv(f'{config.data_dir}/train_targets_nonscored.csv')
test_features = pd.read_csv(f'{config.data_dir}/test_features.csv')
submission = pd.read_csv(f'{config.data_dir}/sample_submission.csv')
# Feature Engineering
train_features, test_features = preprocess_fn(train_features, test_features, train_targets_scored, config)
# Preprocess Data
if config.remove_vehicle:
train_features = train_features[train_features.cp_type != 'ctl_vehicle']