def main():
utils.heading('SETUP')
config = configure.Config(mode=FLAGS.mode, model_name=FLAGS.model_name)
config.write()
with tf.Graph().as_default() as graph:
model_trainer = trainer.Trainer(config)
summary_writer = tf.summary.FileWriter(config.summaries_dir)
checkpoints_saver = tf.train.Saver(max_to_keep=1)
best_model_saver = tf.train.Saver(max_to_keep=1)
init_op = tf.global_variables_initializer()
graph.finalize()
with tf.Session() as sess:
sess.run(init_op)
progress = training_progress.TrainingProgress(
config, sess, checkpoints_saver, best_model_saver,
config.mode == 'train')
utils.log()
if config.mode == 'train':
utils.heading('START TRAINING ({:})'.format(config.model_name))
model_trainer.train(sess, progress, summary_writer)
elif config.mode == 'eval':
utils.heading('RUN EVALUATION ({:})'.format(config.model_name))
progress.best_model_saver.restore(
sess, tf.train.latest_checkpoint(config.checkpoints_dir))
model_trainer.evaluate_all_tasks(sess, summary_writer, None)
else:
raise ValueError('Mode must be "train" or "eval"')
def trainGan():
ecalData = np.load('resources/ecaldata/%s.npz' % datasetName)
dataSet = torch.utils.data.TensorDataset(
torch.from_numpy(ecalData['EnergyDeposit']).float())
dataLoader = torch.utils.data.DataLoader(dataSet,
batch_size=batch_size,
shuffle=True,
num_workers=1)
lossCalculator = training.losses.GanLoss(device, problem, nn.BCELoss()) if type == mygan.GANS.GAN \
else (training.losses.WganLoss if type == mygan.GANS.WGAN else training.losses.CramerEneryGanLoss)(problem,
training.losses.GradientPenalizer(gpWeight,True,ngpu > 0))
ganTrainer = trainer.Trainer(device, problem, lossCalculator,
initOptimizer, lambda d: d[0], ganFile)
ui = plotUi.ShowPlotUi()
painter = painters.ECalPainter(ui)
print("Starting Training Loop...")
dopt, gopt = ganTrainer.train(netD, netG, dataLoader, num_epochs,
hyperParams, painter, 9)
# painter.plotFake(netG.forward(torch.randn(128000, nz, 1, 1, device=device)), num_epochs, 0)
ui.toView(
lambda: painters.plotLosses(ganTrainer.G_losses, ganTrainer.D_losses))
if type != mygan.GANS.GAN:
ui.toView(lambda: painters.plotGradPenalties(
ganTrainer.ganLoss.gradientPenalizer.penalties, ganTrainer.ganLoss.
gradientPenalizer.norms))
ui.close()
def run(self, cmd_args):
# TODO: add window counts and feature labels if preprocessing is skipped
feature_files = cmd_args.input
window_counts = None
feature_labels = None
rec_names = None
self.stats = stats.Stats()
# --------------------------------------------------- PREPROCESSING ---------------------------------------------------#
if not cmd_args.no_pre:
self.preprocessor = preprocessor.Preprocessor(self.stats)
feature_files, window_counts, self.stats = self.preprocessor.preprocess(
cmd_args)
else:
logging.info("\t\tPreprocessing is skipped.")
if not cmd_args.pre_only:
if cmd_args.no_pre and cmd_args.window != 'boxcar' or cmd_args.windowsize != 2 or cmd_args.overlap != 0 or \
cmd_args.bands != '1,4,8,12,18,24,30,60,90':
logging.warning(
"\t\tYou cannot change window, window_size, overlap or bands when not doing "
"preprocessing. These are feature computation settings. In this mode you are not "
"computing new features but using already computed features."
)
cmd_args = my_io.update_cmd_args(cmd_args, feature_files)
logging.info(
'\tUpdated the cmd arguments to: {}.'.format(cmd_args))
# -------------------------------------------- TRAINING / CV / AUTOSKLEARN --------------------------------------------#
self.trainer = trainer.Trainer(jobs=cmd_args.n_proc,
auto_skl=cmd_args.auto_skl)
self.trainer.train(cmd_args, window_counts)
# TODO: return the classifier, parameters that lead to result that can be used for prediction on test set
results = self.trainer.get_results()
(accs, precs, recs, f1s) = results
logging.info("\t\t\tMean accuracy is {:.2f} ({:.2f}).".format(
accs[1], accs[2]))
if len(cmd_args.input) == 2:
logging.info("\t\t\tMean precision is {:.2f} ({:.2f}).".format(
precs[1], precs[2]))
logging.info("\t\t\tMean recall is {:.2f} ({:.2f}).".format(
recs[1], recs[2]))
logging.info("\t\t\tMean f1 score is {:.2f} ({:.2f}).".format(
f1s[1], f1s[2]))
# ---------------------------------------------------- PREDICTION -----------------------------------------------------#
# self.predictor = predictor.Predictor()
# self.predictor.predict(feature_files)
# -------------------------------------------- POSTPROCESSING / ANALYSIS ----------------------------------------------#
# self.postprocessor = postprocessor.Postprocessor(feature_files, window_counts, rec_names)
# self.postprocessor.postprocess(cmd_args)
else:
logging.info(
"\t\tTraining, prediction and postprocessing is skipped.")
if not cmd_args.no_pre:
self.stats.log()
def setUp(self):
self.trnr = trainer.Trainer()
self.trnr.folds = 3
self.a = np.arange(24, dtype=float).reshape(2, 3, 4)
""" this is unreal feature data. it has 2 classes x 3 recordings x 4 values
np.array([
[[0, 1, 2, 3],
[4, 5, 6, 7],
[8, 9, 10, 11]],
[[12, 13, 14, 15],
[16, 17, 18, 19],
[20, 21, 22, 23]]
])
"""
# this is needed since without it it comes to confusion when predicting the first fold
self.a[1] *= 2
self.train_folds, self.test_folds = self.trnr.create_folds(self.a)
import auxiliary.argument_parser as argument_parser
import auxiliary.my_utils as my_utils
import time
import torch
from auxiliary.my_utils import yellow_print
"""
Main training script.
author : Thibault Groueix 01.11.2019
"""
opt = argument_parser.parser()
torch.cuda.set_device(opt.multi_gpu[0])
my_utils.plant_seeds(random_seed=opt.random_seed)
import training.trainer as trainer
trainer = trainer.Trainer(opt)
trainer.build_dataset()
trainer.build_network()
trainer.build_optimizer()
trainer.build_losses()
trainer.start_train_time = time.time()
if opt.demo:
with torch.no_grad():
trainer.demo(opt.demo_input_path)
sys.exit(0)
if opt.run_single_eval:
with torch.no_grad():
trainer.test_epoch()
sys.exit(0)
# assert np.all(np.diff(stage_schedule) > 0)
lambda_schedule[stage] = stage_schedule
learning_rates = {}
for k, v in config["learning_rates"].items():
learning_rates[k] = np.array([float(x) for x in v.split(",")])
engine = trainer.Trainer(
host_pdbfile,
stubs,
worker_address_list,
ff_handlers,
lambda_schedule,
int(general_cfg["du_dl_cutoff"]),
float(restr_config["search_radius"]),
float(restr_config["force_constant"]),
int(general_cfg["n_frames"]),
int(intg_cfg["steps"]),
float(intg_cfg["dt"]),
float(intg_cfg["temperature"]),
float(intg_cfg["friction"]),
learning_rates,
general_cfg["precision"],
)
for epoch in range(100):
print("Starting Epoch", epoch,
datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
epoch_dir = os.path.join(general_cfg["out_dir"], "epoch_" + str(epoch))
from sklearn.linear_model import ElasticNet
from training import trainer
# Set up trainer which initializes train/test sets
trainer = trainer.Trainer()
model = ElasticNet(random_state=10)
# Set up param grid
param_grid = [{
'alpha': [0, 0.001, 0.01, 0.1, 1, 10],
'fit_intercept': [False, True],
'l1_ratio': [0.1, 0.5, 0.9],
'max_iter': [100, 300, 750, 1000, 2000],
'positive': [True, False],
'selection': ['cyclic', 'random']
}]
# Train
trained_grid = trainer.train(model, param_grid)
predictor = trained_grid.best_estimator_
best_params = trained_grid.best_params_
best_score = trained_grid.best_score_
print("The best parameters found are", best_params)
print("The best RMSE score was", best_score)
# Test
trainer.test(predictor)
# Save results
params_file = 'elasticnet.txt'
pickle_file = 'elasticnet.pkl'
def initNet(netClass):
net = netClass(type, hyperParams, problem).to(device)
# Handle multi-gpu if desired
if (device.type == 'cuda') and (ngpu > 1):
net = nn.DataParallel(net, list(range(ngpu)))
net.apply(mygan.weights_init)
print(net) # Print the model
return net
# Initialize BCELoss function; preprocess is my own extension of torch.Dataset
lossCalculator = training.losses.GanLoss(device, problem, nn.BCELoss()) if type == mygan.GANS.GAN \
else (training.losses.WganLoss if type == mygan.GANS.WGAN else training.losses.CramerGanLoss)(problem, training.losses.GradientPenalizer(gpWeight, True, ngpu > 0))
ganTrainer = trainer.Trainer(device, problem, lossCalculator, initOptimizer,
dataSet.preprocess, 'resources/norm07142030.pth')
netG = initNet(multimodalnorm.ganModelMultimodal.Generator)
netD = initNet(multimodalnorm.ganModelMultimodal.Discriminator)
print("Starting Training Loop...")
ganTrainer.train(netD, netG, dataLoader, num_epochs, hyperParams, painter,
12800)
painter.plotFake(netG.forward(torch.randn(128000, nz, 1, 1, device=device)),
num_epochs, 0)
painters.plotLosses(ganTrainer.G_losses, ganTrainer.D_losses)
if type != mygan.GANS.GAN:
painters.plotGradPenalties(ganTrainer.ganLoss.gradientPenalizer.penalties,
ganTrainer.ganLoss.gradientPenalizer.norms)
assert np.all(np.diff(stage_schedule) > 0)
else:
raise Exception("unknown stage")
# stage 1 and 2 must be monotonically increasing
# assert np.all(np.diff(stage_schedule) > 0)
lambda_schedule[stage] = stage_schedule
learning_rates = {}
for k, v in config['learning_rates'].items():
learning_rates[k] = np.array([float(x) for x in v.split(',')])
engine = trainer.Trainer(host_pdbfile, stubs, worker_address_list,
ff_handlers, lambda_schedule,
int(general_cfg['du_dl_cutoff']),
float(restr_config['search_radius']),
float(restr_config['force_constant']),
int(general_cfg['n_frames']),
int(intg_cfg['steps']), float(intg_cfg['dt']),
float(intg_cfg['temperature']),
float(intg_cfg['friction']), learning_rates,
general_cfg['precision'])
for epoch in range(100):
print("Starting Epoch", epoch,
datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
epoch_dir = os.path.join(general_cfg['out_dir'], "epoch_" + str(epoch))
if not os.path.exists(epoch_dir):
os.makedirs(epoch_dir)
epoch_params = serialize_handlers(ff_handlers)
else:
# stage 1 and 2 must be monotonically increasing
assert np.all(np.diff(stage_schedule) > 0)
lambda_schedule.append(stage_schedule)
restr_cfg = config['restraints']
intg_cfg = config['integrator']
lr_config = config['learning_rates']
engine = trainer.Trainer(host_pdbfile, stubs, worker_address_list,
ff_handlers, lambda_schedule,
int(general_cfg['du_dl_cutoff']),
restr_cfg['core_smarts'],
int(general_cfg['n_frames']),
float(restr_cfg['force']),
float(restr_cfg['alpha']),
int(restr_cfg['count']), int(intg_cfg['steps']),
float(intg_cfg['dt']),
float(intg_cfg['temperature']),
float(intg_cfg['friction']),
float(lr_config['charge']),
general_cfg['precision'])
for epoch in range(100):
print("Starting Epoch", epoch,
datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
epoch_dir = os.path.join(general_cfg['out_dir'], "epoch_" + str(epoch))
if not os.path.exists(epoch_dir):
os.makedirs(epoch_dir)
def main():
utils.heading('SETUP')
config = configure.Config(mode=FLAGS.mode, model_name=FLAGS.model_name)
config.write()
if config.mode == 'encode':
word_vocab = embeddings.get_word_vocab(config)
sentence = "Squirrels , for example , would show up , look for the peanut , go away .".split()
sentence = ([word_vocab[embeddings.normalize_word(w)] for w in sentence])
print(sentence)
return
if config.mode == 'decode':
word_vocab_reversed = embeddings.get_word_vocab_reversed(config)
sentence = "25709 33 42 879 33 86 304 92 33 676 42 32 13406 33 273 445 34".split()
sentence = ([word_vocab_reversed[int(w)] for w in sentence])
print(sentence)
return
if config.mode == 'encode-vi':
word_vocab_vi = embeddings.get_word_vocab_vi(config)
print(len(word_vocab_vi))
sentence = "Mỗi_một khoa_học_gia đều thuộc một nhóm nghiên_cứu , và mỗi nhóm đều nghiên_cứu rất nhiều đề_tài đa_dạng .".split()
sentence = ([word_vocab_vi[embeddings.normalize_word(w)] for w in sentence])
print(sentence)
return
if config.mode == 'decode-vi':
word_vocab_reversed_vi = embeddings.get_word_vocab_reversed_vi(config)
sentence = "8976 32085 129 178 17 261 381 5 7 195 261 129 381 60 37 2474 1903 6".split()
sentence = ([word_vocab_reversed_vi[int(w)] for w in sentence])
print(sentence)
return
if config.mode == 'embed':
word_embeddings = embeddings.get_word_embeddings(config)
word = 50
embed = word_embeddings[word]
print(' '.join(str(x) for x in embed))
return
if config.mode == 'embed-vi':
word_embeddings_vi = embeddings.get_word_embeddings_vi(config)
word = 50
embed = word_embeddings_vi[word]
print(' '.join(str(x) for x in embed))
return
with tf.Graph().as_default() as graph:
model_trainer = trainer.Trainer(config)
summary_writer = tf.summary.FileWriter(config.summaries_dir)
checkpoints_saver = tf.train.Saver(max_to_keep=1)
best_model_saver = tf.train.Saver(max_to_keep=1)
init_op = tf.global_variables_initializer()
graph.finalize()
with tf.Session() as sess:
sess.run(init_op)
progress = training_progress.TrainingProgress(
config, sess, checkpoints_saver, best_model_saver,
config.mode == 'train')
utils.log()
if config.mode == 'train':
#summary_writer.add_graph(sess.graph)
utils.heading('START TRAINING ({:})'.format(config.model_name))
model_trainer.train(sess, progress, summary_writer)
elif config.mode == 'eval-train':
utils.heading('RUN EVALUATION ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.evaluate_all_tasks(sess, summary_writer, None, train_set=True)
elif config.mode == 'eval-dev':
utils.heading('RUN EVALUATION ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.evaluate_all_tasks(sess, summary_writer, None, train_set=False)
elif config.mode == 'infer':
utils.heading('START INFER ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.infer(sess)
elif config.mode == 'translate':
utils.heading('START TRANSLATE ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.translate(sess)
elif config.mode == 'eval-translate-train':
utils.heading('RUN EVALUATION ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.evaluate_all_tasks(sess, summary_writer, None, train_set=True, is_translate=True)
elif config.mode == 'eval-translate-dev':
utils.heading('RUN EVALUATION ({:})'.format(config.model_name))
progress.best_model_saver.restore(sess, tf.train.latest_checkpoint(
config.checkpoints_dir))
model_trainer.evaluate_all_tasks(sess, summary_writer, None, train_set=False, is_translate=True)
else:
raise ValueError('Mode must be "train" or "eval"')
