def prepare_model(
stations: typing.Dict[typing.AnyStr, typing.Tuple[float, float, float]],
target_time_offsets: typing.List[datetime.timedelta],
config: typing.Dict[typing.AnyStr, typing.Any],
) -> tf.keras.Model:
"""This function should be modified in order to prepare & return your own prediction model.
See https://github.com/mila-iqia/ift6759/tree/master/projects/project1/evaluation.md for more information.
"""
################################### MODIFY BELOW ##################################
# Team 7 model factory
factory = ModelFactory(stations, target_time_offsets, config)
return factory.load_model_from_config()
def __init__(self, logger):
self.logger = logger
self.index_factory = IndexFactory()
self.example_reader_factory = ExampleReaderFactory(self.index_factory)
self.example_extender_factory = ExampleExtenderFactory(
self.index_factory)
self.example_batcher_factory = ExampleBatcherFactory()
self.model_factory = ModelFactory(self.index_factory)
self.model_tester_factory = ModelTesterFactory(
self.example_reader_factory, self.example_extender_factory,
self.example_batcher_factory, self.index_factory)
self.model_trainer_factory = ModelTrainerFactory(
self.example_reader_factory, self.example_extender_factory,
self.example_batcher_factory, self.model_tester_factory,
self.logger)
def train():
run_this_model = available_models[2]
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def kfold_validate(self, splits, repeat):
"""
Returns predictions from a repeated k-fold split
Used for statistical tests
"""
model_names = ModelFactory.get_models_list()
result = pd.DataFrame(columns=["actual"] + model_names)
for i in range(0, repeat):
kFold = KFold(n_splits=splits, shuffle=True, random_state=None)
for train_index, test_index in kFold.split(self.data):
train_data = self.data.iloc[train_index]
test_data = self.data.iloc[test_index]
y_train = train_data['non-information']
y_test = test_data['non-information']
features_test = self.extract_features(test_data)
features_train = self.extract_features(train_data)
features_test = self.combine_features(features_test,
comments_only=True)
features_train = self.combine_features(features_train,
comments_only=True)
data = {'actual': y_test.tolist()}
for model_name in model_names:
y_pred = self.execute_model_data(model_name,
features_train, y_train,
features_test)
data[model_name] = y_pred.tolist()
df = pd.DataFrame(data=data)
result = result.append(df)
return result
def train():
model_name = available_models[0]
language_model = ModelFactory.make_model(model_name)
hyperparams = net_conf.get_hyperparams(model_name)
corpus_name = available_corpus[2]
corpus_params = params.get_corpus_params(corpus_name)
tools.train_model(language_model, hyperparams, corpus_params)
def main():
run_this_model = available_models[0]
language_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.batch_size = 1
corpus_name = available_corpus[0]
corpus_params = params.get_corpus_params(corpus_name)
tools.train_model(language_model, hyperparams, corpus_params)
def main():
run_this_model = available_models[2]
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.batch_size = 1
dataset_name = available_datasets[2]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def train(params, log):
# specify dataset
data = DatasetFactory.create(params)
# specify model
model = ModelFactory.create(params)
# define loss function (criterion)
criterion = set_loss_function(params)
# optimizer & scheduler & load from checkpoint
optimizer, scheduler, start_epoch, best_prec = set_optimizer_scheduler(params, model, log)
# log details
log_string = "\n" + "==== NET MODEL:\n" + str(model)
log_string += "\n" + "==== OPTIMIZER:\n" + str(optimizer) + "\n"
log_string += "\n" + "==== SCHEDULER:\n" + str(scheduler) + "\n"
log_string += "\n" + "==== DATASET (TRAIN):\n" + str(data.dataset['train']) + "\n"
log_string += "\n" + "==== DATASET (VAL):\n" + repr(data.dataset['val']) + "\n"
log.log_global(log_string)
# train
for epoch in range(start_epoch, params['TRAIN']['epochs']):
# train for one epoch
_, _ = train_epoch(data.loader['train'], model, criterion, optimizer, scheduler, epoch,
params['device'], log)
# evaluate on train set
acc_train, loss_train = validate(data.loader['train'], model, criterion, params['device'])
# evaluate on validation set
acc_val, loss_val = validate(data.loader['val'], model, criterion, params['device'])
# remember best prec@1
is_best = acc_val > best_prec
best_prec = max(acc_val, best_prec)
# save checkpoint
if params['LOG']['do_checkpoint']:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec': best_prec,
'optimizer': optimizer.state_dict(),
'scheduler': scheduler
}, model, params, is_best)
# logging results
time_string = log.timers['global'].current2str() # get current time
log.log_epoch(epoch + 1,
acc_train, loss_train,
acc_val, loss_val,
is_best, time_string)
def apply():
model_name = available_models[1]
text_match_model = ModelFactory.make_model(model_name)
model_url = ''
text_match_model.load(model_url)
hyperparams = net_conf.get_hyperparams(model_name)
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
text_match_model.setup(hyperparams, dataset_params)
text_match_model.evaluate_generator()
def tune_enc_layer_num_TEBLDModel():
run_this_model = available_models[2]
model_full_name = model_name_abbr_full[run_this_model]
print('============ ' + model_full_name + ' tune enc layer num ============')
enc_layer_nums = [1, 2, 3, 4, 5, 6]
for layer_num in enc_layer_nums:
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.layers_num = layer_num
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def ensemble_max_vote(features_train_text,
features_train_notext,
features_test_text,
features_test_notext,
y_train,
y_test,
imbalance_sampling=None):
model_notext = ModelFactory.get_model('LogisticRegression')
model_text = ModelFactory.get_model('RandomForest')
model_text.fit_model(features_train_text, y_train)
model_notext.fit_model(features_train_notext, y_train)
pred1 = model_text.predict(features_test_text)
pred2 = model_notext.predict(features_test_notext)
final_pred = np.array([])
for i in range(0, len(y_test)):
val = mode([pred1[i], pred2[i]])
final_pred = np.append(final_pred, val)
return ScoreMetrics.get_scores(y_test, final_pred)
def tune_layer_num_SBLDModel():
run_this_model = available_models[1]
model_full_name = model_name_abbr_full[run_this_model]
print('============ ' + model_full_name + ' tune layer num ============')
# RNMTPlusEncoderBiLSTMDenseModel | StackedBiLSTMDenseModel
layer_nums = [0, 1, 2, 3]
for num in layer_nums:
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.bilstm_retseq_layer_num = num
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def tune_dropout_rate_REBLDModel():
model_name = available_models[3]
model_full_name = model_name_abbr_full[model_name]
print('============ ' + model_full_name + ' tune dropout rate ============')
p_dropouts = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
for p_dropout in p_dropouts:
text_match_model = ModelFactory.make_model(model_name)
hyperparams = net_conf.get_hyperparams(model_name)
hyperparams.lstm_p_dropout = p_dropout
hyperparams.dense_p_dropout = p_dropout
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def compare_models(self, features_train, features_test, y_train, y_test):
"""
Executes models of all types implemented in this project and prints their results
"""
x_train = features_train
model_names = ModelFactory.get_models_list()
score_df = pd.DataFrame(
columns=['name', 'accuracy', 'precision', 'recall', 'f1'])
if self.imbalance_sampling:
x_train, y_train = ImbalanceSampling.get_sampled_data(
self.imbalance_sampling, x_train, y_train)
for name in model_names:
model = ModelFactory.get_model(name, optimised=False)
model.fit_model(x_train, y_train)
y_pred = model.predict(features_test)
score = ScoreMetrics.get_scores(name, y_test, y_pred)
print('-------')
print(name)
ScoreMetrics.print_scores(y_test, y_pred)
score_df = score_df.append(score)
return score_df
def ensemble_averaging(features_train_text,
features_train_notext,
features_test_text,
features_test_notext,
y_train,
y_test,
w1,
w2,
imbalance_sampling=None):
model_notext = ModelFactory.get_model('SVM')
model_text = ModelFactory.get_model('RandomForest', optimised=False)
y_train_text = y_train
if imbalance_sampling != None:
features_train_text, y_train_text = ImbalanceSampling.get_sampled_data(
imbalance_sampling, features_train_text, y_train)
features_train_notext, y_train = ImbalanceSampling.get_sampled_data(
imbalance_sampling, features_train_notext, y_train)
# models that perform best with imbalance sampling
model_notext = ModelFactory.get_model('MLP', optimised=True)
model_text = ModelFactory.get_model('SVM', optimised=False)
model_notext.fit_model(features_train_notext, y_train)
model_text.fit_model(features_train_text, y_train_text)
pred1 = model_notext.predict_proba(features_test_notext)
pred2 = model_text.predict_proba(features_test_text)
final_pred = np.array([])
for i in range(0, len(y_test)):
first = (w1 * pred1[i][0] + w2 * (pred2[i][0]))
second = (w1 * pred1[i][1] + w2 * (pred2[i][1]))
val = 1
if first > second:
val = 0
final_pred = np.append(final_pred, val)
return ScoreMetrics.get_scores('ensemble', y_test, final_pred)
def tune_dropout_rate_DULBModel():
model_name = available_models[0]
model_full_name = model_name_abbr_full[model_name]
print('============ ' + model_full_name +
' tune dropout rate ============')
# Don't set dropout rate too large, because it will cause information loss.
p_dropouts = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
for p_dropout in p_dropouts:
language_model = ModelFactory.make_model(model_name)
hyperparams = net_conf.get_hyperparams(model_name)
hyperparams.p_dropout = p_dropout
corpus_name = available_corpus[2]
corpus_params = params.get_corpus_params(corpus_name)
tools.train_model(language_model, hyperparams, corpus_params)
def apply():
model_name = available_models[1]
language_model = ModelFactory.make_model(model_name)
hyperparams = net_conf.get_hyperparams(model_name)
corpus_name = available_corpus[0]
corpus_params = params.get_corpus_params(corpus_name)
model_url = 'E:\PyCharmProjects\yangzl-lm-keras\\result' \
'\REBModel_just_for_test_2018-10-19 00_38_16\epoch_0014-0.99687.h5'
hyperparams.batch_size = 1
language_model.setup(hyperparams, corpus_params)
language_model.build()
language_model.load(model_url)
language_model.compile()
language_model.evaluate_generator()
def tune_state_dim_SBLDModel():
run_this_model = available_models[1]
model_full_name = model_name_abbr_full[run_this_model]
print('============ ' + model_full_name + ' tune hidden state dim num ============')
# RNMTPlusEncoderBiLSTMDenseModel | StackedBiLSTMDenseModel
# The hidden state dim of LSTM should have a certain relationship with the word emb dim.
# Information will be lost if dim is set to small.
state_dims = [100, 200, 300, 400, 500, 600, 700]
for state_dim in state_dims:
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.state_dim = state_dim
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def execute_model(self, name, imbalance_sampling=None):
model = ModelFactory.get_model(name)
split_data = self.split_data()
features_train = self.extract_features(split_data['x_train'])
features_test = self.extract_features(split_data['x_test'])
features_train = self.combine_features(features_train, False)
features_test = self.combine_features(features_test, False)
x_train = features_train
y_train = split_data['y_train']
if imbalance_sampling:
x_train, y_train = ImbalanceSampling.get_sampled_data(
imbalance_sampling, x_train, y_train)
model.fit_model(x_train, y_train)
y_pred = model.predict(features_test)
ScoreMetrics.print_scores(split_data['y_test'], y_pred)
return model
def tune_dropout_rate_SBLDModel():
model_name = available_models[1]
model_full_name = model_name_abbr_full[model_name]
print('============ ' + model_full_name + ' tune dropout rate ============')
# Don't set dropout rate too large, because it will cause information loss.
# According to previous experiment: lstm rate >= 0.5, 0 <= dense rate <= 0.2
lstm_p_dropouts = [0.5, 0.6, 0.7]
dense_p_dropouts = [0, 0.1, 0.2]
for lstm_rate in lstm_p_dropouts:
for dense_rate in dense_p_dropouts:
text_match_model = ModelFactory.make_model(model_name)
hyperparams = net_conf.get_hyperparams(model_name)
hyperparams.lstm_p_dropout = lstm_rate
hyperparams.dense_p_dropout = dense_rate
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
class ExperimentFactory:
example_reader_factory = None
example_extender_factory = None
index_factory = None
model_factory = None
model_updater_factory = None
def __init__(self, logger):
self.logger = logger
self.index_factory = IndexFactory()
self.example_reader_factory = ExampleReaderFactory(self.index_factory)
self.example_extender_factory = ExampleExtenderFactory(
self.index_factory)
self.example_batcher_factory = ExampleBatcherFactory()
self.model_factory = ModelFactory(self.index_factory)
self.model_tester_factory = ModelTesterFactory(
self.example_reader_factory, self.example_extender_factory,
self.example_batcher_factory, self.index_factory)
self.model_trainer_factory = ModelTrainerFactory(
self.example_reader_factory, self.example_extender_factory,
self.example_batcher_factory, self.model_tester_factory,
self.logger)
def get(self, experiment_configuration):
self.logger.write("Building experiment:",
area="context",
subject="configuration")
self.logger.write_configuration(experiment_configuration,
area="context",
subject="configuration")
model_trainer = self.model_trainer_factory.get(
experiment_configuration)
model_tester = self.model_tester_factory.get(experiment_configuration)
model = self.model_factory.get(experiment_configuration)
experiment = Experiment(model_trainer, model_tester, model,
self.logger)
return experiment
def tune_l2_lambda_SBLDModel():
run_this_model = available_models[1]
model_full_name = model_name_abbr_full[run_this_model]
print('============ ' + model_full_name + ' tune l2 lambda ============')
# RNMTPlusEncoderBiLSTMDenseModel | StackedBiLSTMDenseModel
kernel_l2_lambdas = [1e-5, 1e-4]
recurrent_l2_lambdas = [1e-5, 1e-4]
bias_l2_lambdas = [1e-5, 1e-4]
activity_l2_lambdas = [0, 1e-5, 1e-4]
for kernel_l2_lambda in kernel_l2_lambdas:
for recurrent_l2_lambda in recurrent_l2_lambdas:
for bias_l2_lambda in bias_l2_lambdas:
for activity_l2_lambda in activity_l2_lambdas:
text_match_model = ModelFactory.make_model(run_this_model)
hyperparams = net_conf.get_hyperparams(run_this_model)
hyperparams.kernel_l2_lambda = kernel_l2_lambda
hyperparams.recurrent_l2_lambda = recurrent_l2_lambda
hyperparams.bias_l2_lambda = bias_l2_lambda
hyperparams.activity_l2_lambda = activity_l2_lambda
dataset_name = available_datasets[0]
dataset_params = params.get_dataset_params(dataset_name)
tools.train_model(text_match_model, hyperparams, dataset_params)
def train_and_eval_from_config(common_config, model_config, model_output_dir):
# for reproducibility
tf.set_random_seed(0)
# Load Data from tfrecord
# first get the reader based on the config: frame feature reader or
# aggregated feature reader
reader = get_reader(common_config['feature_names'],
common_config['feature_sizes'],
common_config['num_classes'],
common_config['frame_features'])
# get the input data tensors
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_training_tensors(
reader,
common_config['training_data_path'],
batch_size=common_config['batch_size'],
num_readers=common_config['num_readers'],
num_epochs=common_config['num_epochs']
))
tf.summary.histogram("model/input_raw", model_input_raw)
# create model
model_name = next(iter(model_config))
train_model = ModelFactory().get_model(model_name).create_model(model_input_raw,
common_config,
model_config)
# compile the model
# Pass the target tensor `labels_batch` to train_model.compile
# via the `target_tensors` keyword argument
classifier = common_config['classifier']
if classifier == 'multiclass':
train_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'],
target_tensors=[labels_batch])
elif classifier == 'binary':
train_model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'],
target_tensors=[labels_batch])
train_model.summary()
# get the evaluation data tensors
video_id_batch, eval_model_input_raw, eval_labels_batch, eval_num_frames = (
get_input_evaluation_tensors(
reader,
common_config['evaluation_data_path'],
batch_size=common_config['batch_size'],
num_readers=common_config['num_readers']))
tf.summary.histogram("eval/input_raw", eval_model_input_raw)
# create a separate model for evaluation
eval_model = ModelFactory().get_model(model_name).create_model(eval_model_input_raw,
common_config,
model_config)
# compile the eval model
# Pass the target tensor `eval_labels_batch` to eval_model.compile
# via the `target_tensors` keyword argument
if classifier == 'multiclass':
eval_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'],
target_tensors=[eval_labels_batch])
elif classifier == 'binary':
eval_model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'],
target_tensors=[eval_labels_batch])
eval_model.summary()
# get the session..
from keras import backend as K
sess = K.get_session()
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())
# Fit the model using data from the TFRecord data tensors.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
train_model.fit(epochs=common_config['num_epochs'],
steps_per_epoch=common_config['training_steps_per_epoch'],
callbacks=[EvaluateInputTensor(eval_model, steps=common_config['eval_steps_per_epoch'])],
verbose=2)
except tf.errors.OutOfRangeError:
logging.info("Done training -- epoch limit reached.")
# save the model
# 1. serialize model to JSON
model_json = train_model.to_json()
model_file_to_save = os.path.join(model_output_dir, OUTPUT_MODEL_FILENAME)
with open(model_file_to_save, "w") as json_file:
json_file.write(model_json)
# 2. save the model weights
weights_file_to_save = os.path.join(model_output_dir, OUTPUT_WEIGHTS_FILENAME)
train_model.save_weights(weights_file_to_save)
logging.info("Saved model and weights to " + model_output_dir)
# Clean up the TF session.
coord.request_stop()
coord.join(threads)
K.clear_session()
def initialize(self):
model_factory = ModelFactory()
self.model = model_factory.get_model()
def train(params, log, time_keeper):
# specify dataset
data = DatasetFactory.create(params)
# specify model
model = ModelFactory.create(params)
model = model.to(params['device'])
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().to(params['device'])
optimizer = torch.optim.SGD(model.parameters(),
lr=params['TRAIN']['lr'],
momentum=params['TRAIN']['momentum'])
# resume from a checkpoint
if len(params['TRAIN']['resume']) > 0:
start_epoch, best_prec = load_checkpoint(log, model,
params['TRAIN']['resume'],
optimizer)
else:
start_epoch = 0
best_prec = 0
# scheduler (if any)
if 'lr_schedule_step' in params['TRAIN']:
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=params['TRAIN']['lr_schedule_step'],
gamma=params['TRAIN']['lr_schedule_gamma'])
else:
scheduler = None
# log details
log_string = "\n" + "==== NET MODEL:\n" + str(model)
log_string += "\n" + "==== OPTIMIZER:\n" + str(optimizer) + "\n"
log.log_global(log_string)
time_keeper.start()
# train
for epoch in range(start_epoch, params['TRAIN']['epochs']):
# adjust_learning_rate
if scheduler:
scheduler.step()
# train for one epoch
_, _ = train_epoch(data.loader['train'], model, criterion, optimizer,
epoch, params['device'], log, timer)
# evaluate on train set
acc_train, loss_train = validate(data.loader['train'], model,
criterion, params['device'])
# evaluate on validation set
acc_val, loss_val = validate(data.loader['val'], model, criterion,
params['device'])
# remember best prec@1 and save checkpoint
is_best = acc_val > best_prec
best_prec = max(acc_val, best_prec)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec': best_prec,
'optimizer': optimizer.state_dict(),
'scheduler': scheduler
}, model, params, is_best)
# logging results
time_string = time_keeper.get_current_str() # get current time
log.log_epoch(epoch + 1, acc_train, loss_train, acc_val, loss_val,
is_best, time_string)
import pandas as pd
from models.model_exec import ModelExec
exec = ModelExec(include_comments=False, include_long_code=True)
from mlxtend.evaluate import paired_ttest_5x2cv
from statsmodels.stats.contingency_tables import mcnemar,cochrans_q
from models.model_factory import ModelFactory
# executes McNemar test to check the variance
# between results from k-fold validation of different models
model_names = ModelFactory.get_models_list()
result = exec.kfold_validate(2, 10)
data = []
for i in range(0, len(model_names)):
name1 = model_names[i]
df = pd.DataFrame()
for j in range(0, len(model_names)):
name2 = model_names[j]
if name1 != name2:
yes_yes = 0
no_no = 0
yes_no = 0
no_yes = 0
for index, row in result.iterrows():
r1 = row[name1]
r2 = row[name2]
actual = row['actual']
def train(params, log, time_keeper, tboard_exp_path):
writer = SummaryWriter(tboard_exp_path)
# specify dataset
data = DatasetFactory.create(params)
# specify model
model = ModelFactory.create(params)
model = model.to(params['device'])
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().to(params['device'])
if params['MODEL']['name'] == 'resnet18':
optimizer = torch.optim.SGD(model.fc.parameters(),
lr=params['TRAIN']['lr'],
momentum=params['TRAIN']['momentum'])
elif params['MODEL']['name'] == 'vgg19':
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=params['TRAIN']['lr'],
momentum=params['TRAIN']['momentum'])
else:
optimizer = torch.optim.SGD(model.parameters(),
lr=params['TRAIN']['lr'],
momentum=params['TRAIN']['momentum'])
# DRAFT FOR EXPERIMENT
exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=6,
gamma=0.1)
# resume from a checkpoint
if len(params['TRAIN']['resume']) > 0:
start_epoch, best_prec = load_checkpoint(log, model,
params['TRAIN']['resume'],
optimizer)
else:
start_epoch = 0
best_prec = 0
# scheduler (if any)
if 'lr_schedule_step' in params['TRAIN']:
scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=params['TRAIN']['lr_schedule_step'],
gamma=params['TRAIN']['lr_schedule_gamma'])
else:
scheduler = None
# log details
log_string = "\n" + "==== NET MODEL:\n" + str(model)
log_string += "\n" + "==== OPTIMIZER:\n" + str(optimizer) + "\n"
log.log_global(log_string)
time_keeper.start()
# train
for epoch in range(start_epoch, params['TRAIN']['epochs']):
# adjust_learning_rate
if scheduler:
scheduler.step()
# train for one epoch
_, _ = train_epoch(data.loader['train'], model, criterion, optimizer,
epoch, params['device'], log, timer, writer,
exp_lr_scheduler)
# evaluate on train set
acc_train, loss_train = validate(data.loader['train'], model,
criterion, params['device'])
# evaluate on validation set
acc_val, loss_val = validate(data.loader['val'], model, criterion,
params['device'])
correct = 0
total = 0
class_correct = list(0. for i in range(len(classes)))
class_total = list(0. for i in range(len(classes)))
hoodie_not_correct = []
with torch.no_grad():
for (input_data, target) in data.loader['train']:
images = input_data.to(params['device'])
labels = target.to(params['device'])
outputs = model(images)
_, predicted = torch.max(outputs.data, 1)
c = (predicted == labels).squeeze()
total += labels.size(0)
correct += (predicted == labels).sum().item()
for j in range(4):
label = labels[j]
class_correct[label] += c[j].item()
class_total[label] += 1
print('Accuracy of the network on the % epoch test images: %d %%' %
(epoch, 100 * correct / total))
for k in range(len(classes)):
print('Accuracy of %5s : %2d %%' %
(classes[k], 100 * class_correct[k] / class_total[k]))
writer.add_scalar('Accuracy of %5s' % classes[k],
(100 * class_correct[k] / class_total[k]),
epoch + 1)
# remember best prec@1 and save checkpoint
is_best = acc_val > best_prec
best_prec = max(acc_val, best_prec)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec': best_prec,
'optimizer': optimizer.state_dict(),
'scheduler': scheduler
}, model, params, is_best)
# logging results
time_string = time_keeper.get_current_str() # get current time
log.log_epoch(epoch + 1, acc_train, loss_train, acc_val, loss_val,
is_best, time_string)
writer.add_scalar("Train accuracy ", acc_train, epoch + 1)
writer.add_scalar("Train Loss ", loss_train, epoch + 1)
writer.add_scalar("Test accuracy ", acc_val, epoch + 1)
writer.add_scalar("Test Loss ", loss_val, epoch + 1)
exp_lr_scheduler.step()
# Fix the random seed of Pytorch when using GPU.
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.random_state)
torch.cuda.manual_seed(args.random_state)
# Fix the random seed of Pytorch when using CPU.
torch.manual_seed(args.random_state)
torch.random.manual_seed(args.random_state)
# get dataset and alphabets
dataset = DataIOSST2(config['data'])
if config['use_pre_embedding']:
seq_alphabet = AlphabetEmbeddings(**config['embedding'])
seq_alphabet.load_embeddings_from_file()
else:
seq_alphabet = AlphabetEmbeddings(**config['embedding'])
seq_alphabet.add_instance(dataset.train_word)
label_alphabet = Alphabet('label', False, False)
label_alphabet.add_instance(dataset.train_label)
# get model
if args.load is not None:
model = torch.load(args.load)
else:
model = ModelFactory.get_model(config, args, seq_alphabet,
label_alphabet)
process = Process(config, args, dataset, model, seq_alphabet,
label_alphabet)
process.train()
def main():
"""
Does the following:
- For each subject:
- Load preprocessed data from subject (preprocessed from 'A0XT.mat')
- Train model on ALL data from 'A0XT.mat'
- Evaluate model on test data originating from 'A0XE.mat'
"""
parser = argparse.ArgumentParser(
description="Train and run model for data set 2a of BCI Competition IV."
)
parser.add_argument(
"-m",
"--model_name",
type=str,
help="Name of the model used",
default="eegA",
choices=["eegA", "eegB", "eegC", "eegD", "eegA_LSTM", "eegD_LSTM"])
parser.add_argument("-cf", "--num_conv_filters", type=int, default=32)
parser.add_argument(
'--stride',
dest='stride',
help="Whether stride is used in the last Conv2D of first block",
action='store_true')
parser.add_argument('--no-stride', dest='stride', action='store_false')
parser.set_defaults(stride=True)
parser.add_argument("-dr", "--dropout_rate", type=float, default=0.5)
parser.add_argument("-bs", "--batch_size", type=int, default=16)
parser.add_argument("-e", "--epochs", type=int, default=10)
parser.add_argument("-p",
"--patience",
help="Parameter for EarlyStopping callback",
type=int,
default=10)
parser.add_argument("-kf", "--k_fold", type=int, default=5)
parser.add_argument(
"-o",
"--output_name",
type=str,
help="logs will be put in ./logs/fit/output_name. If none is"
"provided, time at run start is chosen",
default=None)
args = parser.parse_args()
# input validation
try:
num_windows = json.load(open("./data/args_bci2aiv_preprocess.txt",
'r'))['num_windows']
except FileNotFoundError:
raise FileNotFoundError(
"Preprocessed data arguments not found. Run main_preprocess_data_bci2aiv.py and try again."
)
if num_windows == 1 and 'LSTM' in args.model_name:
raise ValueError(
"LSTM can only be chosen for data preprocessed with -w > 1")
if num_windows > 1 and 'LSTM' not in args.model_name:
raise ValueError(
"Only LSTM models can be chosen for data preprocessed with -w > 1")
if args.output_name is None:
args.output_name = datetime.now().strftime('%Y%m%d-%H%M%S')
model_factory = ModelFactory(dataset="BCI2aIV",
output_name=args.output_name,
model_name=args.model_name,
num_conv_filters=args.num_conv_filters,
use_stride=args.stride,
dropout_rate=args.dropout_rate
) # num_classes is always 4 for this dataset
for subject_num in g.subject_num_range_bci2aiv:
X_train, y_train = load_single_subject_bci2aiv_data(
subject_num=subject_num, is_training=True)
X_test, y_test = load_single_subject_bci2aiv_data(
subject_num=subject_num, is_training=False)
X_train, scaler = scale_data(X_train)
X_test, _ = scale_data(X_test, scaler)
model = model_factory.get_model()
history = model.fit(x=X_train,
y=y_train,
batch_size=args.batch_size,
epochs=args.epochs,
validation_data=(X_test, y_test),
callbacks=model_factory.get_callbacks(
patience=args.patience,
log_dir_suffix=f"{subject_num}"),
shuffle=True)
write_history(history.history,
subject_num=subject_num,
log_dir=model_factory.get_log_dir())
with open(f"{model_factory.get_log_dir()}/model_summary.txt",
'w') as file:
model.summary(print_fn=lambda x: file.write(x + '\n'))
# write parameters used for training
with open(f"{model_factory.get_log_dir()}/input_args.txt",
'w') as file:
file.write(json.dumps(args.__dict__, indent=4))
def main():
parser = argparse.ArgumentParser(
description=
"Train and run k-fold cross-validation on physionet BCI 2000 dataset")
parser.add_argument("-c",
"--num_classes",
type=int,
default=4,
choices=[2, 3, 4])
parser.add_argument(
"-m",
"--model_name",
type=str,
help="Name of the model used",
default="eegA",
choices=["eegA", "eegB", "eegC", "eegD", "eegA_LSTM", "eegD_LSTM"])
parser.add_argument("-cf", "--num_conv_filters", type=int, default=32)
parser.add_argument(
'--stride',
dest='stride',
help="Whether stride is used in the last Conv2D of first block",
action='store_true')
parser.add_argument('--no-stride', dest='stride', action='store_false')
parser.set_defaults(stride=True)
parser.add_argument("-dr", "--dropout_rate", type=float, default=0.5)
parser.add_argument("-bs", "--batch_size", type=int, default=16)
parser.add_argument("-e", "--epochs", type=int, default=10)
parser.add_argument("-p",
"--patience",
help="Parameter for EarlyStopping callback",
type=int,
default=5)
parser.add_argument("-kf", "--k_fold", type=int, default=5)
parser.add_argument(
"-o",
"--output_name",
type=str,
help="logs will be put in ./logs/fit/output_name. If none is"
"provided, time at run start is chosen",
default=None)
args = parser.parse_args()
# input validation
try:
num_windows = json.load(open("./data/args_bci2000_preprocess.txt",
'r'))['num_windows']
except FileNotFoundError:
raise FileNotFoundError(
"Preprocessed data arguments not found. Run main_preprocess_data_bci2000.py and try again."
)
if num_windows == 1 and 'LSTM' in args.model_name:
raise ValueError(
"LSTM can only be chosen for data preprocessed with -w > 1")
if num_windows > 1 and 'LSTM' not in args.model_name:
raise ValueError(
"Only LSTM models can be chosen for data preprocessed with -w > 1")
if args.output_name is None:
args.output_name = datetime.now().strftime('%Y%m%d-%H%M%S')
model_factory = ModelFactory(dataset="BCI2000",
output_name=args.output_name,
model_name=args.model_name,
num_classes=args.num_classes,
num_conv_filters=args.num_conv_filters,
dropout_rate=args.dropout_rate,
use_stride=args.stride)
X, y = load_preprocessed_bci2000_data(num_classes=args.num_classes)
kf = KFold(n_splits=args.k_fold, shuffle=True, random_state=42)
for idx, [train, test] in enumerate(kf.split(X, y)):
X_train = X[train]
X_test = X[test]
y_train = y[train]
y_test = y[test]
X_train, scaler = scale_data(X_train)
X_test, _ = scale_data(X_test, scaler)
model = model_factory.get_model()
history = model.fit(x=X_train,
y=y_train,
batch_size=args.batch_size,
epochs=args.epochs,
validation_data=(X_test, y_test),
callbacks=model_factory.get_callbacks(
patience=args.patience,
log_dir_suffix=f"{idx + 1}"),
shuffle=True)
write_history(history.history, log_dir=model_factory.get_log_dir())
with open(f"{model_factory.get_log_dir()}/model_summary.txt",
'w') as file:
model.summary(print_fn=lambda x: file.write(x + '\n'))
# write parameters used for training
with open(f"{model_factory.get_log_dir()}/input_args.txt",
'w') as file:
file.write(json.dumps(args.__dict__, indent=4))
if "seed_numpy" not in param_interface["training"] or param_interface[
"training"]["seed_numpy"] == -1:
seed = randrange(0, 2**32)
param_interface["training"].add_custom_params({"seed_numpy": seed})
logger.info("Setting numpy random seed to: {}".format(
param_interface["training"]["seed_numpy"]))
np.random.seed(param_interface["training"]["seed_numpy"])
# Initialize the application state singleton.
app_state = AppState()
# check if CUDA is available turn it on
check_and_set_cuda(param_interface['training'], logger)
# Build the model.
model = ModelFactory.build_model(param_interface['model'])
model.cuda() if app_state.use_CUDA else None
# Log the model summary.
logger.info(model.summarize())
# Build problem for the training
problem = ProblemFactory.build_problem(
param_interface['training']['problem'])
if 'curriculum_learning' in param_interface['training']:
# Initialize curriculum learning - with values from config.
problem.curriculum_learning_initialize(
param_interface['training']['curriculum_learning'])
# Set initial values of curriculum learning.
curric_done = problem.curriculum_learning_update_params(0)