def train_fold( model: ModelInterface, train_data: List[Item], sampler: util.Sampler, val_batch: List[Any], val_label: List[int], *, no_reset: bool, learning_rate: float, batch_size: int, beta: float, epsilon: float, score_expression: Text, maximal_count: int, min_iteration: int, max_iteration: int, **kwargs ): if no_reset: log.warn('Model resetting disabled.') else: model.reset() log.debug('in preprocessing...') model.set_mode(training=True) model.preprocess(train_data) optimizer = torch.optim.Adam(params=model.params(), lr=learning_rate) watcher = util.MaximalCounter() # training iterations min_loss = 1e99 # track history minimal loss batch_per_epoch = ceil(len(train_data) / batch_size) log.debug(f'batch_per_epoch={batch_per_epoch}') for i in range(max_iteration): # epochs sum_loss = 0.0 epoch_start = time.time() for _ in range(batch_per_epoch): # batches # generate batch batch, _label = util.separate_items(sampler.get_batch()) label = torch.tensor(_label) # train a mini-batch batch_loss = train_step(model, optimizer, batch, label) sum_loss += batch_loss loss = sum_loss / batch_per_epoch roc_auc, prc_auc, pred_label = evaluate_model(model, val_batch, val_label) f_score = util.metrics.fbeta_score(val_label, pred_label, beta=beta) watcher.record(eval(score_expression, None, { 'prc_auc': prc_auc, 'roc_auc': roc_auc, 'f_score': f_score, 'loss': loss })) time_used = time.time() - epoch_start log.debug(f'[{i}] train: loss={loss},min={min_loss}') log.debug(f'[{i}] validate: {util.stat_string(val_label, pred_label)}. roc={roc_auc},prc={prc_auc},fβ={f_score}') log.debug(f'[{i}] watcher: {watcher}') log.debug(f'[{i}] epoch time={"%.3f" % time_used}s') # if i >= min_iteration and abs(min_loss - loss) < epsilon: # break # save state if watcher.is_updated(): model.save_checkpoint() if i >= min_iteration and watcher.count >= maximal_count: break min_loss = min(min_loss, loss) sum_loss = 0.0 # load best model model.load_checkpoint() log.debug('in postprocessing...') model.postprocess(train_data) model.set_mode(training=False)