def record(self, fold):
# save plots
save_plot(self.val_record, 'loss', self.args.n_eval,
'tmp/val_loss.png')
save_plot(self.val_record, 'f1', self.args.n_eval, 'tmp/val_f1.png')
save_plot(self.norm_record, 'grad_norm', self.args.n_eval,
'tmp/grad_norm.png')
if self.args.test:
save_plots([self.val_record, self.test_record], ['loss', 'f1'],
['val', 'test'], self.args.n_eval)
# create subdir for this experiment
os.makedirs(self.record_dir, exist_ok=True)
subdir = os.path.join(self.models_dir, str_date_time())
if self.args.mode == 'test':
subdir += '_test'
os.mkdir(subdir)
# write model params and results to csv
csvlog = os.path.join(subdir, 'info.csv')
param_dict = {}
for arg in vars(self.args):
param_dict[arg] = str(getattr(self.args, arg))
info = torch.load(self.best_info_path)
hash = get_hash() if self.args.machine == 'dt' else 'no_hash'
passed_args = ' '.join(sys.argv[1:])
param_dict = {
'hash': hash,
'subdir': subdir,
**param_dict,
**info, 'args': passed_args
}
dict_to_csv(param_dict, csvlog, 'w', 'index', reverse=False)
header = True if fold == 0 else False
dict_to_csv(param_dict,
self.record_path,
'a',
'columns',
reverse=True,
header=header)
# copy all records to subdir
png_files = ['val_loss.png', 'val_f1.png'
] if not self.args.test else ['loss.png', 'f1.png']
csv_files = [
'val_probs*.csv', 'train_steps.csv', 'submission.csv',
'test_probs.csv'
]
copy_files([*png_files, 'models/*.info', *csv_files], 'tmp', subdir)
return subdir
def train(model, config, logger, options, model_dir, samples_dir,
inverse=False, save_step=10, n_samples=20):
utils.makedirs(samples_dir)
losses = defaultdict(list)
for epoch in tqdm(range(options.n_epochs), file=sys.stdout):
logger.info("Epoch {}".format(epoch))
samples_generator = config.data_iter(
options.data, options.b_size, inverse=inverse, n_samples=n_samples)
entropy_epoch = []
for it in tqdm(range(options.n_iters), file=sys.stdout):
Z_global = None
if inverse:
Z_global = np.random.uniform(
-1., 1., (options.b_size, config.nz_global, 1, 1))
Z_samples = utils.sample_noise_tensor(
config, options.b_size, config.zx, global_noise=Z_global)
X_samples = next(samples_generator)
if it % (config.k + 1) != 0:
if inverse == 0:
losses['G_iter'].append(model.train_g(Z_samples))
elif inverse == 1:
losses['G_iter'].append(model.train_g(X_samples, Z_samples,
Z_global))
elif inverse >= 2:
# losses['G_iter'].append(model.train_g(X_samples[0],
# Z_samples))
G_loss, entropy = model.train_g(X_samples[0], Z_samples)
losses['G_iter'].append(G_loss)
entropy_epoch.append(entropy)
else:
if inverse == 0:
losses['D_iter'].append(model.train_d(X_samples, Z_samples))
elif inverse == 1:
losses['D_iter'].append(model.train_d(X_samples, Z_samples,
Z_global))
elif inverse >= 2:
losses['D_iter'].append(model.train_d(
X_samples[0], X_samples[1], Z_samples))
msg = "Gloss = {}, Dloss = {}"
msg += "\n e_min = {}, e_max = {}, e_mean = {}, e_med = {}"
e_min, e_max, e_mean, e_med = [f(entropy_epoch) for f in
[np.min, np.max, np.mean, np.median]]
losses['G_epoch'].append(np.mean(losses['G_iter'][-options.n_iters:]))
losses['D_epoch'].append(np.mean(losses['D_iter'][-options.n_iters:]))
logger.info(msg.format(losses['G_epoch'][-1], losses['D_epoch'][-1],
e_min, e_max, e_mean, e_med))
X = next(samples_generator)
real_samples, gen_samples, large_sample = utils.sample_after_iteration(
model, X, inverse, config, options.b_size)
utils.save_samples(
samples_dir, [real_samples, gen_samples, large_sample],
['real', 'gen', 'large'], epoch=epoch)
utils.save_plots(samples_dir, losses, epoch, options.n_iters)
if (epoch+1) % save_step == 0:
model_file = 'epoch_{:04d}.model'.format(epoch)
model.save(os.path.join(model_dir, model_file))
def main(data_path, out_report_path, random_state, tune_params):
# read the data files, split into X and y
print("Start build_model script")
random_state = int(random_state)
tune = True if tune_params == 'True' else False
if tune:
print("We will tune the hyperparameters")
else:
print("We will use the predefined hyperamater values")
print("Read the data files, split into X and y")
X_train, y_train, X_test, y_test = sdr.read_data_as_xy(data_path)
preprocessor = make_column_transformer(
("drop", sdr.drop_features),
(StandardScaler(), sdr.numerical_features),
(OneHotEncoder(handle_unknown="ignore"), sdr.categorical_features),
(OneHotEncoder(handle_unknown="error",
drop="if_binary"), sdr.binary_features),
)
# tuning hyperparameters for SVC, RandomForestClassifier and
# LogisticRegression
print("Process models")
hyperparams_best_model = tune_hyperparams(preprocessor, X_train, y_train,
random_state, tune)
# find the best model
print("Finding the best model using cross validation")
_, best_model, _ = find_best_model(
hyperparams_best_model,
X_train,
y_train,
random_state,
data_path + "/model_selection_result.csv",
)
# get result plots
print("Creating plots and classification report")
plot, class_report = utils.plot_results(best_model, X_test, y_test,
["No-revenue", "Revenue"])
# save plots to report path
print("Saving reports")
utils.save_plots(
out_report_path,
plot,
class_report,
("confusion_matrix", "classification_report"),
)
# save model to disk
print("Saving the best model for later use")
pickle.dump(best_model, open(data_path + "/best_model.sav", "wb"))
# try feature selection
print("Building the model with RFE")
fs_model = make_pipeline(
preprocessor,
RFECV(Ridge(random_state=random_state), cv=10),
RandomForestClassifier(max_depth=12, n_estimators=275),
)
fs_model.fit(X_train, y_train)
plot, class_report = utils.plot_results(fs_model, X_test, y_test,
["No-revenue", "Revenue"])
# save plots to report path
print("Saving reports")
utils.save_plots(
out_report_path,
plot,
class_report,
(
"confusion_matrix_feature_selection",
"classification_report_feature_selection",
),
)
print("End build_model script")
return
opt)
# Save
CP = shelf(dataQ=dataQ,
dataP=dataP,
wP=wP,
collQ=collQ,
coll_mmd=coll_mmd,
opt=opt)
CP._save(FILENAME, date=False)
# Save plot
save_plots(
collQ,
coll_mmd,
opt.plottimes,
dataP,
wP,
filename=FILENAME,
alpha_target=0.2,
caption=r"$\tau=${}, $\alpha=${}, bsD={}, $n_c$={}, layers={}".format(
opt.tau, opt.alpha, opt.batchSizeD, opt.n_c, opt.layerSizes))
# Save animation
if opt.save_animation:
save_animation(collQ,
coll_mmd,
caption=r"$\tau=${}, $\alpha=${}".format(
opt.tau, opt.alpha),
filename=FILENAME)
train_data['Generator Cost'].append(
[mb_idx, output_g['batch_cost']])
train_data['Log Gradient Norm'].append(
[mb_idx, np.log10(output_d['grad_norm'])])
# print losses
msg = ("Disc. loss: {:.2f}, Gen. loss: {:.2f}, Grad Norm: {:.2f}")
progress_bar.set_description(
msg.format(float(output_d['batch_cost']),
float(output_g['batch_cost']),
float(output_d['grad_norm'])))
# generate plots
# save some examples from dataset
if mb_idx == 0:
save_plots(image_samp,
'mnist-examples.png',
train_data,
args,
title='mnist data examples')
# save generated images
if mb_idx % args.plot_interval == 0:
output_g = train_computation_g({'noise': z_samp})
generated = output_g['generated']
pltname = '{}-mnist-iter-{}.png'.format(args.loss_type, mb_idx)
title = 'iteration {} disc_cost{:.2E} gen_cost{:.2E}'.format(
mb_idx, float(output_d['batch_cost']),
float(output_g['batch_cost']))
save_plots(generated, pltname, train_data, args, title=title)
[mb_idx, np.log10(output_d['grad_norm'])])
# print losses
msg = (
"Discriminator avg loss: {:.5f} Generator avg loss: {:.5f} Gradient Norm: {:.5f}"
)
progress_bar.set_description(
msg.format(float(output_d['batch_cost']),
float(output_g['batch_cost']),
float(output_d['grad_norm'])))
# plot some examples from dataset
if mb_idx == 0:
save_plots(image_samp,
'lsun-examples.png',
train_data,
args,
title='lsun bedroom data examples')
# display some generated images
if mb_idx % args.plot_interval == 0:
generated = output_g['generated'] # N, C, H, W
if args.loss_type == "WGAN":
prefix = 'Originial'
elif args.loss_type == "WGAN-GP":
prefix = 'Improved'
pltname = '{}-wgan-lsun-iter-{}.png'.format(prefix, mb_idx)
title = 'iteration {} disc_cost{:.2E} gen_cost{:.2E}'.format(
mb_idx, float(output_d['batch_cost']),
float(output_g['batch_cost']))
save_plots(generated, pltname, train_data, args, title=title)