def experiment():
X, y = load_data("./data/wdbc.data")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train_std = scaler.fit_transform(X_train)
X_test_std = scaler.transform(X_test)
hparam_grid = {"C": 10 ** np.linspace(-3, 1, 10)}
opt_model = select_hyperparameters(LogisticRegression(random_state=42),
hparam_grid,
X_train_std, y_train,
"cvresults.csv")
evaluate_model(opt_model, X_test_std, y_test, "best_model")
def report_scores(model: ModelInterface, batch: List[object],
label: List[int]) -> Tuple[float, float]:
roc_auc, prc_auc, pred_label = evaluate_model(model,
batch,
label,
show_stats=True)
log.info(f'test: {util.stat_string(label, pred_label)}')
log.info(f'ROC-AUC: {roc_auc}')
log.info(f'PRC-AUC: {prc_auc}')
return roc_auc, prc_auc
def main(config):
##### EXPERIMENT #####
savedir = experiment_mkdir(config)
##### DATASET #####
db = TCGDB.TCGDB(config["path"]["tcg_root"])
db.open()
db.set_subclasses(
train_subclasses=int(config["training-mode"]["subclasses"]))
print("Finished opening TCG dataset.")
##### LEARNING #####
for run_id in range(
db.get_number_runs(
protocol_type=config["training-mode"]["protocol_type"])):
X_train, Y_train, X_test, Y_test = db.get_train_test_data(
run_id=run_id,
protocol_type=config["training-mode"]["protocol_type"])
print(
"\n{:10}: ".format("Run"),
db.get_train_test_set(
run_id=run_id,
protocol_type=config["training-mode"]["protocol_type"]))
print("{:10}: ".format("X_train"), X_train.shape,
" | {:10}: ".format("Y_train"), Y_train.shape)
print("{:10}: ".format("X_test"), X_test.shape,
" | {:10}: ".format("Y_test"), Y_test.shape)
model = train.fit_model(config, X_train, Y_train, X_test, Y_test,
savedir, str(run_id))
train.evaluate_model(config, model, X_test, Y_test, savedir,
str(run_id))
def main(opts):
"""Main function for the training pipeline
:opts: commandline arguments
:returns: None
"""
pprint(vars(opts))
##########################################################################
# Basic settings #
##########################################################################
exp_dir = 'experiments'
log_dir = os.path.join(exp_dir, 'logs')
model_dir = os.path.join(exp_dir, 'models')
os.makedirs(model_dir, exist_ok=True)
os.makedirs(os.path.join(log_dir, opts.run_name), exist_ok=True)
with open(os.path.join(log_dir, opts.run_name, "args.json"), 'w') as f:
json.dump(vars(opts), f, indent=True)
##########################################################################
# Define all the necessary variables for model training and evaluation #
##########################################################################
writer = SummaryWriter(os.path.join(log_dir, opts.run_name))
train_dataset = SegmentationDataset(is_train=True)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opts.batch_size,
num_workers=4,
drop_last=False,
shuffle=True)
val_dataset = SegmentationDataset(is_train=False)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=20,
shuffle=False,
num_workers=0,
drop_last=False)
model = load_model(opts, n_classes=4)
if torch.cuda.is_available():
model = model.cuda()
optimizer = optim.Adam(model.parameters(), lr=opts.lr, weight_decay=0.1)
if opts.lr_scheduler == "plateau":
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
patience=opts.patience,
factor=.3,
threshold=0.1,
verbose=True)
elif opts.lr_scheduler == "step":
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=opts.gamma)
best_val_loss = float('inf')
best_val_accu = float(0)
iteration_change_loss = 0
t_start_training = time.time()
##########################################################################
# Main training loop #
##########################################################################
for epoch in range(opts.epochs):
t_start = time.time()
############################################################
# The actual training and validation step for each epoch #
############################################################
train_loss, train_metric = train_model(model, train_loader, epoch,
optimizer, writer, opts)
with torch.no_grad():
val_loss, val_metric = evaluate_model(model, val_loader, epoch,
writer, opts)
##############################
# Adjust the learning rate #
##############################
if opts.lr_scheduler == 'plateau':
scheduler.step(val_loss)
elif opts.lr_scheduler == 'step':
scheduler.step()
t_end = time.time()
delta = t_end - t_start
utils.print_epoch_progress(epoch, opts.epochs, train_loss, val_loss,
delta, train_metric, val_metric)
t_end_training = time.time()
print('training took {}s'.format(t_end_training - t_start_training))
########## DATA LOADERS ##########
train_loader, val_loader, test_loader = load_data(DATA_DIR, train_img_ids, val_img_ids, test_img_ids, CONTEXT_SIZE, BATCH_SIZE, NUM_WORKERS, MAX_BG_BOXES)
########## CREATE MODEL & LOSS FN ##########
model = WebObjExtractionNet(ROI_POOL_OUTPUT_SIZE, IMG_HEIGHT, N_CLASSES, BACKBONE, USE_ATTENTION, HIDDEN_DIM, TRAINABLE_CONVNET, DROP_PROB,
USE_BBOX_FEAT, CLASS_NAMES).to(device)
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
criterion = nn.CrossEntropyLoss(reduction='sum').to(device)
########## TRAIN MODEL ##########
train_model(model, train_loader, optimizer, criterion, N_EPOCHS, device, val_loader, EVAL_INTERVAL, log_file, 'ckpt_%d.pth' % args.device)
########## EVALUATE TEST PERFORMANCE ##########
print('Evaluating test data class wise accuracies...')
evaluate_model(model, test_loader, criterion, device, 'TEST', log_file)
with open (test_acc_domainwise_file, 'w') as f:
f.write('Domain,N_examples,%s,%s,%s\n' % (CLASS_NAMES[1], CLASS_NAMES[2], CLASS_NAMES[3]))
print('Evaluating per domain accuracy for %d test domains...' % len(test_domains))
for domain in test_domains:
print('\n---> Domain:', domain)
test_dataset = WebDataset(DATA_DIR, np.loadtxt('%s/domain_wise_imgs/%s.txt' % (SPLIT_DIR, domain), np.int32).reshape(-1), CONTEXT_SIZE, max_bg_boxes=-1)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=NUM_WORKERS, collate_fn=custom_collate_fn, drop_last=False)
per_class_acc = evaluate_model(model, test_loader, criterion, device, 'TEST')
with open (test_acc_domainwise_file, 'a') as f:
f.write('%s,%d,%.2f,%.2f,%.2f\n' % (domain, len(test_dataset), 100*per_class_acc[1], 100*per_class_acc[2], 100*per_class_acc[3]))
def main():
args = parse_args()
current_time = time.strftime("%Y-%m-%d_%H-%M-%S", time.gmtime())
model_name = 'model_{}_stack_{}_feat_{}_g_{}_{}_' if args.prob <= 0 else 'model_aug_{}_stack_{}_feat_{}_g_{}_{}_'
model_name += 'pt_' if args.pre_train else ''
model_name += 'cbam_' if args.use_cbam else ''
model_name += 'unsup_' if args.unsupervise != 0 else ''
model_name += 'norm_' if args.normalized else ''
save_dir = args.save_dir + model_name.format(args.model_type, args.num_stacks, args.num_features, args.gamma, args.loss_type)\
+ current_time + '/'
train_images_dir = PATH + 'train_images/{}.jpg'
train = pd.read_csv(PATH + 'train_fixed.csv') # .sample(n=20).reset_index()
train = remove_out_image_cars(train)
if args.debug:
train = train.iloc[:50, :]
df_train, df_dev = train_test_split(train, test_size=args.val_size, random_state=42)
df_dev.to_csv('val.csv', index=False)
# Augmentation
albu_list = [RandomBrightnessContrast(brightness_limit=(-0.3, 0.3), contrast_limit=(-0.3, 0.3), p=0.3),
RandomGamma(p=0.2), HueSaturationValue(p=0.3), RGBShift(p=0.3), MotionBlur(p=0.1), Blur(p=0.1),
GaussNoise(var_limit=(20, 100), p=0.2),
ChannelShuffle(p=0.2),
#Normalize(mean=[145.3834, 136.9748, 122.7390], std=[95.1996, 94.6686, 85.9170])
]
transform = Compose(albu_list, p=args.prob)
# Create dataset objects
train_dataset = CarDataset(df_train, train_images_dir, sigma=args.sigma, training=True, transform=transform,
normalized=args.normalized)
dev_dataset = CarDataset(df_dev, train_images_dir, sigma=args.sigma, training=False, normalized=args.normalized)
BATCH_SIZE = args.batch_size
# Create data generators - they will produce batches
# transform not using yet
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
# Gets the GPU if there is one, otherwise the cpu
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
print('Running on', torch.cuda.get_device_name(), 'x', torch.cuda.device_count())
if args.checkpoint:
model, start_epoch = load_checkpoints(args)
save_dir = args.checkpoint + '/'
n_epochs = start_epoch + args.epoch
history = pd.read_csv(os.path.join(save_dir, 'history.csv'))
history = history.iloc[: start_epoch + 1]
args.lr = args.lr / 10
else:
start_epoch = 0
n_epochs = args.epoch
if args.model_type == 'UNet':
model = MyUNet(args.num_classes).to(device)
elif args.model_type == 'HG':
model = HourglassNet(nStacks=args.num_stacks, nModules=1, nFeat=args.num_features, nClasses=args.num_classes)
model.cuda()
elif args.model_type == 'HG2':
model = PoseNet(nstack=args.num_stacks, inp_dim=args.num_features,
oup_dim=args.num_classes, use_cbam=args.use_cbam)
model = model.cuda()
if args.num_stacks <= 2 and args.pre_train:
save = torch.load('./weights/checkpoint_2hg.pt')
elif args.pre_train:
save = torch.load('./weights/checkpoint_8hg.pt')
save = save['state_dict']
# print(model)
# print(list(save.keys()))
# print(model.state_dict().keys())
load_my_state_dict(model, save)
del save
elif args.model_type == 'LHG':
heads = {'hm': 8}
model = create_model('hourglass', heads, 256)
model = model.cuda()
if args.pre_train:
model_dir = './weights/ctdet_coco_hg.pth'
load_model(model, model_dir)
elif args.model_type in ['res_34', 'res_50', 'res_101', 'res_152']:
heads = {'hm': 8}
model = create_model(args.model_type, heads, 0)
model = model.cuda()
history = pd.DataFrame()
if torch.cuda.device_count() > 1 and not isinstance(model, nn.DataParallel):
model = nn.DataParallel(model)
print('Number of model parameters: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[5, 10, 15, 20, 25, 30, 35, 40, 45, 50], gamma=0.5)
# exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10) * len(train_loader) // 3, gamma=0.1)
best_loss = 1e6
# save configuration
if not os.path.exists(save_dir):
os.mkdir(save_dir)
with open(save_dir + 'config.txt', 'w') as f:
f.write(str(args))
# unsupervise part
test_images_dir = PATH + 'test_images/{}.jpg'
test = pd.read_csv(PATH + 'sample_submission.csv')
test = test.sample(n=train.shape[0], replace=True)#.reset_index()
transform_test = Compose(albu_list, p=1)
test_dataset = CarDatasetUnsup(test, test_images_dir, sigma=args.sigma, training= args.unsupervise != 0, transform=transform_test,
normalized=args.normalized)
test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
for epoch in range(start_epoch + 1, n_epochs):
torch.cuda.empty_cache()
gc.collect()
train_loss, train_final_loss = train_model(save_dir, model, epoch, train_loader, test_loader, device,
optimizer, history,
args)
best_loss, eval_loss, clf_losses, regr_losses = evaluate_model(model, epoch, dev_loader, device, best_loss, save_dir, history, args)
cur_lr = optimizer.state_dict()['param_groups'][0]['lr']
with open(save_dir + 'log.txt', 'a+') as f:
line = 'Epoch: {}; Train total loss: {:.3f}; Train final loss: {:.3f}; Eval final loss: {:.3f}; Clf loss: {:.3f}; Regr loss: {:.3f}; Best eval loss: {:.3f}; LR: {}\n' \
.format(epoch,
train_loss,
train_final_loss,
eval_loss,
clf_losses,
regr_losses,
best_loss,
cur_lr)
f.write(line)
history.to_csv(save_dir + 'history.csv', index=False)
scheduler.step()
def main(opts):
"""Main function for the training pipeline
:opts: commandlien arguments
:returns: None
"""
##########################################################################
# Basic settings #
##########################################################################
exp_dir = 'experiments'
log_dir = os.path.join(exp_dir, 'logs')
model_dir = os.path.join(exp_dir, 'models')
os.makedirs(os.path.join(model_dir, opts.run_name), exist_ok=True)
os.makedirs(os.path.join(log_dir, opts.run_name))
pprint(vars(opts))
with open(os.path.join(log_dir, opts.run_name, "args.json"), 'w') as f:
json.dump(vars(opts), f, indent=True)
torch.manual_seed(opts.seed)
np.random.seed(opts.seed)
random.seed(opts.seed)
##########################################################################
# Define all the necessary variables for model training and evaluation #
##########################################################################
writer = SummaryWriter(os.path.join(log_dir, opts.run_name), flush_secs=5)
if opts.train_mode == 'combined':
train_dataset = get_train_dataset(opts.data_root, opts, opts.folder1,
opts.folder2, opts.folder3)
elif opts.train_mode == 'oversampling':
train_dataset = get_train_dataset_by_oversampling(
opts.data_root, opts, opts.folder1, opts.folder2, opts.folder3)
elif opts.train_mode == 'pretrain_and_finetune':
train_dataset, finetune_dataset = get_pretrain_and_finetune_datast(
opts.data_root, opts, opts.folder1, opts.folder2, opts.folder3)
finetune_loader = torch.utils.data.DataLoader(
finetune_dataset,
batch_size=opts.batch_size,
num_workers=opts.num_workers,
drop_last=False,
shuffle=True)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opts.batch_size,
num_workers=opts.num_workers,
drop_last=False,
shuffle=True)
val_dataset = get_val_dataset(os.path.join('data', 'val'), opts)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=opts.eval_batch_size,
shuffle=False,
num_workers=opts.num_workers,
drop_last=False)
test_dataset = get_test_dataset(os.path.join('data', 'test'), opts)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=opts.eval_batch_size,
shuffle=False,
num_workers=opts.num_workers,
drop_last=False)
assert train_dataset.class_to_idx == val_dataset.class_to_idx == test_dataset.class_to_idx, "Mapping not correct"
model = get_model(opts)
opts.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if torch.cuda.device_count() > 1 and not opts.no_data_parallel:
model = nn.DataParallel(model)
model = model.to(opts.device)
optimizer = optim.RMSprop(model.parameters(),
lr=opts.lr,
alpha=0.9,
weight_decay=1e-5,
momentum=0.9)
scheduler = get_lr_scheduler(optimizer, opts)
best_val_loss = float('inf')
best_val_accu = float(0)
best_val_rec = float(0)
best_val_prec = float(0)
best_val_f1 = float(0)
best_val_auc = float(0)
iteration_change_loss = 0
t_start_training = time.time()
##########################################################################
# Main training loop #
##########################################################################
for epoch in range(opts.epochs):
current_lr = get_lr(optimizer)
t_start = time.time()
############################################################
# The actual training and validation step for each epoch #
############################################################
train_loss, train_metric = train_model(model, train_loader, optimizer,
opts)
if epoch == opts.finetune_epoch and opts.train_mode == 'pretrain_and_finetune':
train_loader = finetune_loader
optimizer = optim.RMSprop(model.parameters(),
lr=opts.lr,
alpha=0.9,
weight_decay=1e-5,
momentum=0.9)
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=opts.step_size_finetuning,
gamma=opts.gamma)
# Run the validation set
with torch.no_grad():
val_loss, val_metric = evaluate_model(model, val_loader, opts)
##############################
# Write to summary writer #
##############################
train_acc, val_acc = train_metric['accuracy'], val_metric['accuracy']
train_rec, val_rec = train_metric['recalls'], val_metric['recalls']
train_prec, val_prec = train_metric['precisions'], val_metric[
'precisions']
train_f1, val_f1 = train_metric['f1'], val_metric['f1']
train_auc, val_auc = train_metric['auc'], val_metric['auc']
writer.add_scalar('Loss/Train', train_loss, epoch)
writer.add_scalar('Accuracy/Train', train_acc, epoch)
writer.add_scalar('Precision/Train', train_prec, epoch)
writer.add_scalar('Recall/Train', train_rec, epoch)
writer.add_scalar('F1/Train', train_f1, epoch)
writer.add_scalar('AUC/Train', train_auc, epoch)
writer.add_scalar('Loss/Val', val_loss, epoch)
writer.add_scalar('Accuracy/Val', val_acc, epoch)
writer.add_scalar('Precision/Val', val_prec, epoch)
writer.add_scalar('Recall/Val', val_rec, epoch)
writer.add_scalar('F1/Val', val_f1, epoch)
writer.add_scalar('AUC/Val', val_auc, epoch)
##############################
# Adjust the learning rate #
##############################
if opts.lr_scheduler == 'plateau':
scheduler.step(val_loss)
elif opts.lr_scheduler in ['step', 'cosine']:
scheduler.step()
t_end = time.time()
delta = t_end - t_start
print_epoch_progress(epoch, opts.epochs, train_loss, val_loss, delta,
train_metric, val_metric)
iteration_change_loss += 1
print('-' * 30)
if val_acc > best_val_accu:
best_val_accu = val_acc
if bool(opts.save_model):
torch.save(
model.state_dict(),
os.path.join(model_dir, opts.run_name,
'best_state_dict.pth'))
if val_loss < best_val_loss:
best_val_loss = val_loss
iteration_change_loss = 0
if val_rec > best_val_rec:
best_val_rec = val_rec
if val_prec > best_val_prec:
best_val_prec = val_prec
if val_f1 > best_val_f1:
best_val_f1 = val_f1
print(f'The best validation F1-score is now {best_val_f1}')
print(
f'The validation accuracy and AUC are now {val_acc} and {val_auc}'
)
if val_auc > best_val_auc:
best_val_auc = val_auc
if iteration_change_loss == opts.patience and opts.early_stopping:
print(
('Early stopping after {0} iterations without the decrease ' +
'of the val loss').format(iteration_change_loss))
break
t_end_training = time.time()
print(f'training took {t_end_training - t_start_training}s')
print(f'Best validation accuracy: {best_val_accu}')
print(f'Best validation loss: {best_val_loss}')
print(f'Best validation precision: {best_val_prec}')
print(f'Best validation recall: {best_val_rec}')
print(f'Best validation f1: {best_val_f1}')
print(f'Best validation AUC: {best_val_auc}')
with torch.no_grad():
if opts.train_mode in ['combined', 'oversampling']:
model.load_state_dict(
torch.load(
os.path.join(model_dir, opts.run_name,
'best_state_dict.pth')))
test_loss, test_metric = evaluate_model(model, test_loader, opts)
print(f'The best test F1: {test_metric["f1"]}')
print(f'The best test auc: {test_metric["auc"]}')
print(f'The best test accuracy: {test_metric["accuracy"]}')
def main(opts):
"""Main function for the training pipeline
:opts: commandlien arguments
:returns: None
"""
##########################################################################
# Basic settings #
##########################################################################
exp_dir = 'experiments'
log_dir = os.path.join(exp_dir, 'logs')
model_dir = os.path.join(exp_dir, 'models')
os.makedirs(os.path.join(model_dir, opts.run_name), exist_ok=True)
##########################################################################
# Define all the necessary variables for model training and evaluation #
##########################################################################
writer = SummaryWriter(os.path.join(log_dir, opts.run_name), flush_secs=5)
train_dataset = get_train_dataset(root=os.path.join('data', 'train'))
weights = make_weights_for_balanced_classes(train_dataset.imgs,
len(train_dataset.classes))
weights = torch.DoubleTensor(weights)
sampler = torch.utils.data.sampler.WeightedRandomSampler(
weights, len(weights))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opts.batch_size,
num_workers=6,
drop_last=False,
sampler=sampler)
val_dataset = get_val_dataset(root=os.path.join('data', 'val'))
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=opts.batch_size,
shuffle=False,
num_workers=6,
drop_last=False)
assert train_dataset.class_to_idx == val_dataset.class_to_idx, "Mapping not correct"
model = load_baseline(n_classes=2)
if torch.cuda.is_available():
model = model.cuda()
optimizer = optim.Adam(model.parameters(), lr=opts.lr, weight_decay=0.1)
if opts.lr_scheduler == "plateau":
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
patience=3,
factor=.3,
threshold=1e-4,
verbose=True)
elif opts.lr_scheduler == "step":
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=opts.gamma)
best_val_loss = float('inf')
best_val_accu = float(0)
iteration_change_loss = 0
t_start_training = time.time()
##########################################################################
# Main training loop #
##########################################################################
for epoch in range(opts.epochs):
current_lr = get_lr(optimizer)
t_start = time.time()
############################################################
# The actual training and validation step for each epoch #
############################################################
train_loss, train_metric = train_model(model, train_loader, epoch,
opts.epochs, optimizer, writer,
current_lr, opts.log_every)
with torch.no_grad():
val_loss, val_metric = evaluate_model(model, val_loader, epoch,
opts.epochs, writer,
current_lr)
##############################
# Write to summary writer #
##############################
writer.add_scalar('Loss/Train', train_loss, epoch)
writer.add_scalar('Accuracy/Train', train_metric['accuracy'], epoch)
writer.add_scalar('Precision/Train', train_metric['precisions'], epoch)
writer.add_scalar('Recall/Train', train_metric['recalls'], epoch)
writer.add_scalar('F1/Train', train_metric['f1'], epoch)
writer.add_scalar('Loss/Val', val_loss, epoch)
writer.add_scalar('Accuracy/Val', val_metric['accuracy'], epoch)
writer.add_scalar('Precision/Val', val_metric['precisions'], epoch)
writer.add_scalar('Recall/Val', val_metric['recalls'], epoch)
writer.add_scalar('F1/Val', val_metric['f1'], epoch)
##############################
# Adjust the learning rate #
##############################
if opts.lr_scheduler == 'plateau':
scheduler.step(val_loss)
elif opts.lr_scheduler == 'step':
scheduler.step()
t_end = time.time()
delta = t_end - t_start
print_epoch_progress(train_loss, val_loss, delta, train_metric,
val_metric)
iteration_change_loss += 1
print('-' * 30)
train_acc, val_acc = train_metric['accuracy'], val_metric['accuracy']
# file_name = ('val_acc_{}_train_acc_{}_epoch_{}.pth'.
# format(train_acc, val_acc, epoch))
# torch.save(model, os.path.join(model_dir, opts.run_name, file_name))
if val_acc > best_val_accu:
best_val_accu = val_acc
if bool(opts.save_model):
torch.save(model,
os.path.join(model_dir, opts.run_name, 'best.pth'))
if val_loss < best_val_loss:
best_val_loss = val_loss
iteration_change_loss = 0
if iteration_change_loss == opts.patience and opts.early_stopping:
print(
('Early stopping after {0} iterations without the decrease ' +
'of the val loss').format(iteration_change_loss))
break
t_end_training = time.time()
print('training took {}s'.format(t_end_training - t_start_training))
def eval_score(results_file_base,
results_file_ext,
weights_filename,
saliency=False):
'''
Evaluate a trained model on the score dataset
Arguments
- results_file_base: str
Base results file name. Usually includes run_id but leaves out file extension
- results_file_ext: str
Results file extension, exluduing the period (e.g. 'results')
- weights_filename: str
Filename of saved Tensorflow weights
- saliency: bool, default = False
Whether to compute and plot the saliency map
'''
# read results of best run
results_file = results_file_base + '.' + results_file_ext
results_file_dtypes = results_file + '_dtypes'
# dtypes_series = pd.read_csv('dtype_series', header=None)
# dtypes_series = dtypes_series.set_index(0).squeeze()
# dtypes_dict = dtypes_series.to_dict()
df = pd.read_csv(results_file, header=0, float_precision='high',
sep='\t') # dtype=dtypes_dict
series = df.iloc[0]
params = series.to_dict()
# Get data
datasets = ['train', 'test', 'score']
metrics = ['loss', 'acc', 'auroc', 'auroc_sk']
X, Y = train.get_data(params, datasets)
# unpack params
rand_seed = params['rand_seed']
kernel_reg_const = params['kernel_reg_const']
num_features = params['num_features']
q = params['q']
node_array = params['node_array'].split(',')
for i in range(len(node_array)):
node_array[i] = int(node_array[i].strip('[] '))
node_array = np.array(node_array)
# rebuild model
model = models.DNN(num_features, node_array, kernel_reg_const, rand_seed)
# recreate results dict
loss, acc, auroc, auroc_sk, y_prob = {}, {}, {}, {}, {}
for res in [loss, acc, auroc, auroc_sk, y_prob]:
for dataset in datasets:
res[dataset] = []
results = {
'best_index': 0,
'loss': loss,
'acc': acc,
'auroc': auroc,
'auroc_sk': auroc_sk,
'y_prob': y_prob
}
# restore graph
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, weights_filename)
# evaluate model on all datasets, including score
train.evaluate_model(X, Y, model, q, results, datasets, sess)
for dataset in datasets:
y_prob = sess.run(model.y_prob, feed_dict={model.x: X[dataset]})
results['y_prob'][dataset] = y_prob
# plot ROC curve and save results
train.plot_ROC(X, Y, results, datasets, results_file_base)
train.save_results(X, params, results, metrics, datasets,
results_file_base)
# compute and plot saliency map
if saliency:
saliency_vecs = train.saliency(X, Y, model, sess)
train.plot_saliency(saliency_vecs, num_features, results_file_base)
# Model and window configuration
#
# Only run one line in this section!
# ===============================
model, window = model.single_out(dfs, input_steps=25, model_type='lstm')
# model, window = model.multi_out(
# dfs, input_steps=25, out_steps=25, model_type='lstm')
# Load model history
# model, history = train.load_model('model')
# ==============================
# Training
# ===============================
history = train.compile_and_fit(model, window).history
# train.save_model(model, history, 'model')
# ==============================
# Predictions and Evaluations
# ===============================
train.evaluate_model(model, window, history, verbose=2)
window.plot(model, plot_col='t2m')
predict.predict_plot(model, window, label='t2m')