def main(config):
logger = config.get_logger('train')
# fix random seeds for reproducibility
seed_everything(seed=config.config['seed'])
metric_bests = []
# logger = config.get_logger('train')
for i,train_dataloader, valid_dataloader, test_dataloader in makeDataLoader(config):
model = makeModel(config)
# logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader)
trainer = Trainer(model, criterion, metrics, optimizer,
config=config,
i_fold=i,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
metric_bests.append(trainer.mnt_best)
logger.info('metric scores:{}'.format(metric_bests))
logger.info('metric mean score: {}'.format(sum(metric_bests) / float(len(metric_bests))))
def main(config):
from data_process import makeDataLoader
# 针对不同的数据，训练过程的设置略有不同。
# from trainer.weibo_trainer import Trainer # weibo
# from trainer.cnews_trainer import Trainer # cnews
from trainer.medical_question_trainer import Trainer
logger = config.get_logger('train')
train_dataloader, valid_dataloader, test_dataloader = makeDataLoader(config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader.dataset)
trainer = Trainer(model, criterion, metrics, optimizer,
config=config,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
def main(config):
from data_process import makeDataLoader
#
from trainer.htqe_trainer import Trainer
logger = config.get_logger('train')
train_dataloader, valid_dataloader, test_dataloader = makeDataLoader(
config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader.dataset)
trainer = Trainer(model,
criterion,
metrics,
optimizer,
config=config,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
def active_learning(config):
from data_process import makeDataSet
from trainer.trainer import Trainer
logger = config.get_logger('train')
train_set, valid_set, query_set = makeDataSet(config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
# lr_scheduler = makeLrSchedule(config, optimizer, train_set)
trainer = Trainer(
model,
criterion,
metrics,
optimizer,
config=config,
train_dataset=train_set,
valid_dataset=valid_set,
query_dataset=query_set,
test_dataset=None,
# lr_scheduler=lr_scheduler
)
trainer.train()
def main(config):
logger = config.get_logger('train')
# setup data_loader instances
data_loader = config.init_obj('data_loader', module_data)
valid_data_loader = data_loader.split_validation()
# build model architecture, then print to console
model = makeModel(config)
logger.info(model)
# get function handles of loss and metrics
criterion = makeLoss(config)
metrics = makeMetrics(config)
# build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler
optimizer = makeOptimizer(config, model)
lr_scheduler = config.init_obj('lr_scheduler', torch.optim.lr_scheduler,
optimizer)
trainer = Trainer(model,
criterion,
metrics,
optimizer,
config=config,
data_loader=data_loader,
valid_data_loader=valid_data_loader,
lr_scheduler=lr_scheduler)
trainer.train()
def main(config):
logger = config.get_logger('test')
# setup data_loader instances
data_loader = getattr(module_data, config['data_loader']['type'])(
config['data_loader']['args']['data_dir'],
batch_size=512,
shuffle=False,
validation_split=0.0,
training=False,
num_workers=2)
# build model architecture
model = makeModel(config)
logger.info(model)
# get function handles of loss and metrics
loss_fn = makeLoss(config)
metric_fns = makeMetrics(config)
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
total_metrics = torch.zeros(len(metric_fns))
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader)):
data, target = data.to(device), target.to(device)
output = model(data)
#
# save sample images, or do something with output here
#
# computing loss, metrics on test set
loss = loss_fn(output, target)
batch_size = data.shape[0]
total_loss += loss.item() * batch_size
for i, metric in enumerate(metric_fns):
total_metrics[i] += metric(output, target) * batch_size
n_samples = len(data_loader.sampler)
log = {'loss': total_loss / n_samples}
log.update({
met.__name__: total_metrics[i].item() / n_samples
for i, met in enumerate(metric_fns)
})
logger.info(log)