def main(data, label_col, max_depth, n_trees, lr, mlflow_tracking_url,
experiment_name, build_number):
test_data, train_data = reader.load_data(data)
pipeline_model = model.train_model(train_data, label_col, max_depth,
n_trees, lr)
rmse, mae, r2 = model.evaluate_model(pipeline_model, test_data, label_col)
print("Model tree model (max_depth=%f, trees=%f, lr=%f):" %
(max_depth, n_trees, lr))
print(" RMSE: %s" % rmse)
print(" MAE: %s" % mae)
print(" R2: %s" % r2)
with tracking.TrackML(mlflow_tracking_url, experiment_name,
build_number) as track:
track.log_params({
"max_depth": max_depth,
"n_trees": n_trees,
"lr": lr
})
track.log_metrics({"RMSE": rmse, "R2": r2, "MAE": mae})
track.log_model("sklearn", pipeline_model, "retail_model")
def train_and_predict(data_path, data_filename, batch_size, n_epoch):
"""
Create a model, load the data, and train it.
"""
"""
Step 1: Load the data
"""
hdf5_filename = os.path.join(data_path, data_filename)
print("-" * 30)
print("Loading the data from HDF5 file ...")
print("-" * 30)
imgs_train, msks_train, imgs_validation, msks_validation = \
load_data(hdf5_filename)
print("-" * 30)
print("Creating and compiling model ...")
print("-" * 30)
"""
Step 2: Define the model
"""
model = load_model(imgs_train.shape, msks_train.shape)
model_filename, model_callbacks = get_callbacks()
"""
Step 3: Train the model on the data
"""
print("-" * 30)
print("Fitting model with training data ...")
print("-" * 30)
history = model.fit(imgs_train,
msks_train,
batch_size=batch_size,
epochs=n_epoch,
validation_data=(imgs_validation, msks_validation),
verbose=1,
shuffle="batch",
callbacks=model_callbacks)
# Append training log
# with open("training.log","a+") as fp:
# fp.write("{}: {}\n".format(datetime.datetime.now(),
# history.history["val_dice_coef"]))
"""
Step 4: Evaluate the best model
"""
print("-" * 30)
print("Loading the best trained model ...")
print("-" * 30)
model = evaluate_model(model_filename, imgs_validation, msks_validation)
"""
Step 5: Save the best model for inference
"""
print("-" * 30)
print("Saving the model for inference ...")
print("-" * 30)
save_inference_model(model, imgs_train.shape, msks_train.shape)
def train_and_predict(data_path, data_filename, batch_size, n_epoch):
"""
Create a model, load the data, and train it.
"""
"""
Step 1: Load the data
"""
hdf5_filename = os.path.join(data_path, data_filename)
print("-" * 30)
print("Loading the data from HDF5 file ...")
print("-" * 30)
imgs_train, msks_train, imgs_validation, msks_validation = \
load_data(hdf5_filename, args.batch_size,
[args.crop_dim, args.crop_dim])
print("-" * 30)
print("Creating and compiling model ...")
print("-" * 30)
"""
Step 2: Define the model
"""
model = load_model(imgs_train.shape, msks_train.shape)
model_filename, model_callbacks = get_callbacks()
"""
Step 3: Train the model on the data
"""
print("-" * 30)
print("Fitting model with training data ...")
print("-" * 30)
model.fit(imgs_train,
msks_train,
batch_size=batch_size,
epochs=n_epoch,
validation_data=(imgs_validation, msks_validation),
verbose=1,
shuffle="batch",
callbacks=model_callbacks)
"""
Step 4: Evaluate the best model
"""
print("-" * 30)
print("Loading the best trained model ...")
print("-" * 30)
model = evaluate_model(model_filename, imgs_validation, msks_validation)
def test_evaluate_model(self):
"""
Test boundaries of loss and accuracy
"""
_, (mock_X_test, mock_y_test) = datasets.imdb.load_data(num_words=2000)
mock_X_test = sequence.pad_sequences(mock_X_test, maxlen=500)
mock_trained_model = load_model("models/", "mock_trained_model.h5")
loss, acc = evaluate_model(mock_trained_model,
(mock_X_test, mock_y_test))
# Loss
self.assertIsNotNone(loss, "loss not computed")
self.assertGreaterEqual(loss, 0., "loss is negativ")
# Accuracy
self.assertIsNotNone(acc, "accuracy not computed")
self.assertGreaterEqual(acc, 0., "accuracy is negativ")
self.assertLessEqual(acc, 1., "accuracy is greater than 1")
def test_evaluate_model(self):
"""
Test boundaries of loss and accuracy
"""
texts, labels = preprocess_labels(data_dir_path="data/mock_aclImdb",
dataset="test")
vectorized_texts, word_index = tokenize_data(texts)
mock_test_set = (vectorized_texts, labels)
mock_trained_model = load_model("models/", "mock_trained_model.h5")
loss, acc = evaluate_model(mock_trained_model, mock_test_set)
# Loss
self.assertIsNotNone(loss, "loss not computed")
self.assertGreaterEqual(loss, 0., "loss is negativ")
# Accuracy
self.assertIsNotNone(acc, "accuracy not computed")
self.assertGreaterEqual(acc, 0., "accuracy is negativ")
self.assertLessEqual(acc, 1., "accuracy is greater than 1")
from model import evaluate_model
training_data_folder = '/home/girish/dev/sdcnd/CarND-Behavioral-Cloning-P3/train4/'
csv_file_name = 'driving_log.csv'
model_file_name = "model.h5"
csv_file_path = training_data_folder + csv_file_name
test_loss = evaluate_model(model_file_name, training_data_folder,
csv_file_path)
print(test_loss)
def test_is_float(self):
assert isinstance(model.evaluate_model(), float)
def test_is_probability(self):
assert 0 <= model.evaluate_model() <= 1
def main():
download_dataset()
vectorize_dataset()
train_model()
evaluate_model()
def main(_):
if not os.path.exists(FLAGS.save_dir):
os.makedirs(FLAGS.save_dir)
with open(os.path.join(FLAGS.save_dir, 'params.json'), 'w') as f:
f.write(json.dumps(flags.FLAGS.__flags, indent=1))
tr_infoboxes = os.path.join(FLAGS.data_dir, 'train', 'train.box')
tr_sentences = os.path.join(FLAGS.data_dir, 'train', 'train_in.sent')
te_infoboxes = os.path.join(FLAGS.data_dir, 'test', 'test.box')
te_sentences = os.path.join(FLAGS.data_dir, 'test', 'test_in.sent')
va_infoboxes = os.path.join(FLAGS.data_dir, 'valid', 'valid.box')
va_sentences = os.path.join(FLAGS.data_dir, 'valid', 'valid_in.sent')
# Checkpoint directory
checkpoint_dir = os.path.join(FLAGS.save_dir, 'checkpoints')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
batch_size = FLAGS.batch_size
max_source_len = FLAGS.max_source_len
sum_seq_len = FLAGS.sum_seq_length
print("Building the word index")
start = time.time()
if FLAGS.load_pretrained == 1:
targ_path = os.path.join(FLAGS.data_dir, 'target_words.txt')
embed_path = os.path.join(FLAGS.data_dir, 'words.txt')
if FLAGS.trainable == 1:
trainable = True
else:
trainable = False
if os.path.exists(targ_path) and os.path.exists(embed_path):
word_to_id, init_embed = build_index(targ_path, embed_path,
FLAGS.vocab_size)
load_init = True
else:
print('Pretrained embedding not found in the data folder')
return
else:
word_to_id = build_vocabulary(tr_infoboxes, tr_sentences,
FLAGS.vocab_size, FLAGS.min_field_freq,
FLAGS.fields_per_box,
FLAGS.sum_seq_length)
init_embed = None # No use of init_embed when we do not use pretrained embeds
trainable = True
load_init = False
id_to_word = dict(zip(word_to_id.values(), word_to_id.keys()))
vocab_size = len(word_to_id)
duration = time.time() - start
print("Built index in %.3f s" % (duration))
print("Building the training dataset object")
start = time.time()
train_dataset = BaseDataset(tr_infoboxes, tr_sentences,
FLAGS.tokens_per_field, max_source_len,
sum_seq_len, word_to_id, batch_size)
duration = time.time() - start
print("Built train dataset in %.5f s" % (duration))
print("Building the test dataset object")
start = time.time()
test_dataset = BaseDataset(te_infoboxes, te_sentences,
FLAGS.tokens_per_field, max_source_len,
sum_seq_len, word_to_id, batch_size)
duration = time.time() - start
print("Built test dataset in %.5f s" % (duration))
print("Building the valid dataset object")
start = time.time()
valid_dataset = BaseDataset(va_infoboxes, va_sentences,
FLAGS.tokens_per_field, max_source_len,
sum_seq_len, word_to_id, batch_size)
duration = time.time() - start
print("Built valid dataset in %.5f s" % (duration))
with tf.Graph().as_default():
tf.set_random_seed(1234)
model = BaselineSeq2Seq(vocab_size, FLAGS.embedding_size,
FLAGS.learning_rate, FLAGS.optimizer,
FLAGS.rnn_size, init_embed, load_init,
trainable)
enc_inputs = tf.placeholder(tf.int32,
shape=(batch_size, max_source_len),
name="encoder_inputs")
dec_inputs = tf.placeholder(tf.int32,
shape=(batch_size, sum_seq_len + 1),
name="decoder_inputs")
dec_weights = tf.placeholder(tf.float32,
shape=(batch_size, sum_seq_len),
name="decoder_weights")
feed_previous = tf.placeholder(tf.bool, name="feed_previous")
logits_op = model.inference_s2s_att(enc_inputs, dec_inputs,
feed_previous)
loss_op = model.loss(logits_op, dec_inputs, dec_weights)
train_op = model.training(loss_op)
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
init = tf.initialize_all_variables()
sess.run(init)
print "Trainable variables"
print '\n'.join([v.name for v in tf.trainable_variables()])
while train_dataset.epochs_done < FLAGS.num_epochs:
epochs_done = train_dataset.epochs_done
start_e = time.time()
for step in range(train_dataset.num_batches):
benc_ins, bdec_ins, bdec_wts = train_dataset.next_batch()
if train_dataset.epochs_done >= FLAGS.true_feed:
train_feed = True
else:
train_feed = False
feed_dict = {
enc_inputs: benc_ins,
dec_inputs: bdec_ins,
dec_weights: bdec_wts,
feed_previous: train_feed
}
_, loss_val = sess.run([train_op, loss_op],
feed_dict=feed_dict)
perplexity = np.exp(
float(loss_val)) if loss_val < 300 else float('inf')
if step % FLAGS.print_every == 0:
with open(
os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) +
'_train_loss.log'), 'a') as log_f:
log_f.write(
'epoch %d batch %d: loss = %.3f perplexity = %.2f\n'
% (train_dataset.epochs_done + 1, step, loss_val,
perplexity))
print('epoch %d batch %d: loss = %.3f perplexity = %.2f ' %
(train_dataset.epochs_done + 1, step, loss_val,
perplexity))
sys.stdout.flush()
if step % FLAGS.valid_every == 0 and step != 0:
v_loss, v_perp = evaluate_model(sess, valid_dataset,
loss_op, enc_inputs,
dec_inputs, dec_weights,
feed_previous)
with open(os.path.join(FLAGS.save_dir, 'valid.log'),
'a') as log_f:
log_f.write(
'valid : epoch %d batch %d : loss = %0.3f perplexity = %0.3f\n'
% (train_dataset.epochs_done + 1, step, v_loss,
v_perp))
print(
'valid : epoch %d batch %d : loss = %0.3f perplexity = %0.3f\n'
%
(train_dataset.epochs_done + 1, step, v_loss, v_perp))
sys.stdout.flush()
if step % FLAGS.test_every == 0 and step != 0:
t_loss, t_perp = evaluate_model(sess, test_dataset,
loss_op, enc_inputs,
dec_inputs, dec_weights,
feed_previous)
with open(os.path.join(FLAGS.save_dir, 'test.log'),
'a') as log_f:
log_f.write(
'test : epoch %d batch %d : loss = %0.3f perplexity = %0.3f\n'
% (train_dataset.epochs_done + 1, step, t_loss,
t_perp))
print(
'test : epoch %d batch %d : loss = %0.3f perplexity = %0.3f\n'
%
(train_dataset.epochs_done + 1, step, t_loss, t_perp))
sys.stdout.flush()
if step % FLAGS.train_step_every == 0 and step != 0:
epochs_done = train_dataset.epochs_done
index_in_epoch = train_dataset.index_in_epoch
benc_ins, bdec_ins, bdec_wts, sents = train_dataset.next_batch_gen(
)
feed_dict = {
enc_inputs: benc_ins,
dec_inputs: bdec_ins,
dec_weights: bdec_wts,
feed_previous: True
}
logits = np.array(sess.run(logits_op, feed_dict=feed_dict))
logits = np.reshape(
logits,
(FLAGS.batch_size, FLAGS.sum_seq_length, vocab_size))
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_train.gen')
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_train.true')
with open(save_path, 'a') as save_f:
for idx in xrange(batch_size):
words = []
for l in xrange(FLAGS.sum_seq_length):
tokenid = np.argmax(logits[idx, l])
words.append(id_to_word[tokenid])
save_f.write(' '.join(words) + '\n')
with open(true_path, 'a') as true_f:
for sent in sents:
true_f.write(sent)
train_dataset.reset_batch(index_in_epoch, epochs_done)
if step % FLAGS.test_step_every == 0 and step != 0:
benc_ins, bdec_ins, bdec_wts, sents = test_dataset.next_batch_gen(
)
feed_dict = {
enc_inputs: benc_ins,
dec_inputs: bdec_ins,
dec_weights: bdec_wts,
feed_previous: True
}
logits = np.array(sess.run(logits_op, feed_dict=feed_dict))
logits = np.reshape(
logits,
(FLAGS.batch_size, FLAGS.sum_seq_length, vocab_size))
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_test.gen')
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_test.true')
with open(save_path, 'a') as save_f:
for idx in xrange(batch_size):
words = []
for l in xrange(FLAGS.sum_seq_length):
tokenid = np.argmax(logits[idx, l])
words.append(id_to_word[tokenid])
save_f.write(' '.join(words) + '\n')
with open(true_path, 'a') as true_f:
for sent in sents:
true_f.write(sent)
test_dataset.reset_batch()
if step % FLAGS.valid_step_every == 0 and step != 0:
benc_ins, bdec_ins, bdec_wts, sents = valid_dataset.next_batch_gen(
)
feed_dict = {
enc_inputs: benc_ins,
dec_inputs: bdec_ins,
dec_weights: bdec_wts,
feed_previous: True
}
logits = np.array(sess.run(logits_op, feed_dict=feed_dict))
logits = np.reshape(
logits,
(FLAGS.batch_size, FLAGS.sum_seq_length, vocab_size))
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_valid.gen')
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done + 1) + '_' + str(step) +
'_valid.true')
with open(save_path, 'a') as save_f:
for idx in xrange(batch_size):
words = []
for l in xrange(FLAGS.sum_seq_length):
tokenid = np.argmax(logits[idx, l])
words.append(id_to_word[tokenid])
save_f.write(' '.join(words) + '\n')
with open(true_path, 'a') as true_f:
for sent in sents:
true_f.write(sent)
valid_dataset.reset_batch()
if train_dataset.epochs_done != epochs_done + 1:
train_dataset.reset_batch(epochs_completed=epochs_done + 1)
duration_e = time.time() - start_e
with open(os.path.join(FLAGS.save_dir, 'time_taken.txt'),
'a') as time_f:
time_f.write('Epoch : %d\tTime taken : %0.5f\n' %
(train_dataset.epochs_done, duration_e))
if train_dataset.epochs_done % FLAGS.save_every_epochs == 0:
modelfile = os.path.join(
checkpoint_dir,
str(train_dataset.epochs_done) + '.ckpt')
saver.save(sess, modelfile)
if train_dataset.epochs_done % FLAGS.gen_valid_every == 0:
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_valid.true')
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_valid.gen')
generate_sentences(sess,
valid_dataset,
logits_op,
enc_inputs,
dec_inputs,
dec_weights,
feed_previous,
id_to_word,
FLAGS.batch_size,
FLAGS.sum_seq_length,
vocab_size,
save_path,
true_path,
only_num_batches=None)
if train_dataset.epochs_done % FLAGS.gen_train_every == 0:
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_train.true')
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_train.gen')
generate_sentences(sess,
train_dataset,
logits_op,
enc_inputs,
dec_inputs,
dec_weights,
feed_previous,
id_to_word,
FLAGS.batch_size,
FLAGS.sum_seq_length,
vocab_size,
save_path,
true_path,
only_num_batches=10)
if train_dataset.epochs_done % FLAGS.gen_test_every == 0:
true_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_test.true')
save_path = os.path.join(
FLAGS.save_dir,
str(train_dataset.epochs_done) + '_test.gen')
generate_sentences(sess,
test_dataset,
logits_op,
enc_inputs,
dec_inputs,
dec_weights,
feed_previous,
id_to_word,
FLAGS.batch_size,
FLAGS.sum_seq_length,
vocab_size,
save_path,
true_path,
only_num_batches=None)
if __name__ == "__main__":
config = Config()
train_file = "data_labeled"
if len(sys.argv) > 2:
train_file = sys.argv[1]
dataset = Dataset(config)
lines = dataset.read_lines(train_file)
triples = dataset.get_data(lines)
sentence_batches, action_batches = dataset.get_action_batches(
triples, config.batches_num)
model = Transformer(config, len(dataset.vocab))
if torch.cuda.is_available():
model.cuda()
model.train()
optimizer = optim.Adam(model.parameters(), lr=config.lr)
# 多分类交叉熵损失函数
NLLLoss = nn.NLLLoss()
# 模型加载优化函数和损失函数
model.add_optimizer(optimizer)
model.add_loss_op(NLLLoss)
train_losses = []
for i in range(config.max_epochs):
print("Epoch: {}".format(i))
train_loss = model.run_epoch(sentence_batches, action_batches, i)
train_losses.append(train_loss)
train_acc = evaluate_model(model, sentence_batches, action_batches)
print("Final Training Dataset Accuracy: {.4f}".format(train_acc))
'''Faster computation on CPU (only if using tensorflow-gpu)'''
if TENSORFLOW_BACKEND:
print('Switching to TensorFlow for CPU...')
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
if len(sys.argv) == 8:
train_stock_name, val_stock_name, window_size, batch_size, episode_count, model_name, pretrained = sys.argv[1], \
sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]), sys.argv[6], bool(int(sys.argv[7]))
else:
print(
'Usage: python train.py [train stock] [val stock] [window] [batch size] [episodes] [model] [pretrained (0/1)]'
)
exit(0)
agent = Agent(window_size, pretrained=pretrained, model_name=model_name)
train_data = get_stock_data(train_stock_name)
test_data = get_stock_data(val_stock_name)
initial_offset = test_data[1] - test_data[0]
for episode in range(1, episode_count + 1):
train_result = train_model(agent,
episode,
train_data,
episode_count=episode_count,
batch_size=batch_size,
window_size=window_size)
val_result = evaluate_model(agent, test_data, window_size=window_size)
show_train_result(train_result, val_result, initial_offset)
print('Done Training!')
def main(cfg):
"""
# 訓練データと検証データのパス
train_list = make_datapath_list(
csv_file=cfg.csv.train, data_id=cfg.csv.id, data_dir=cfg.data.train_dir
)
val_list = make_datapath_list(
csv_file=cfg.csv.val, data_id=cfg.csv.id, data_dir=cfg.data.val_dir
)
test_list = make_datapath_list(
csv_file=cfg.csv.test, data_id=cfg.csv.id, data_dir=cfg.data.test_dir
)
# 画像表示と確認
size = 224
mean = (0.485, 0.456, 0.406)
std = (0.229, 0.224, 0.225)
img = train_list[100]
img = Image.open(img)
plt.imshow(img)
plt.show()
transform = ImageTransform(size, mean, std)
img_transformed = transform(img)
img_transformed = img_transformed.numpy().transpose((1, 2, 0))
img_transformed = np.clip(img_transformed, 0, 1)
plt.imshow(img_transformed)
plt.show()
# データセットの作成
train_dataset = IsicDataset(
file_list=train_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="train",
csv_file=cfg.csv.train,
label_name=cfg.csv.label,
)
val_dataset = IsicDataset(
file_list=val_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="val",
csv_file=cfg.csv.val,
label_name=cfg.csv.label,
)
test_dataset = IsicDataset(
file_list=test_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="test",
csv_file=cfg.csv.test,
label_name=cfg.csv.label,
)
"""
# Imagedatafolderを用いたデータセットの作成(使わない場合はコメントアウト)
# 学習用データセット作成
train_dataset = make_trainset(
dataroot=cfg.data.train_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 検証用データセット
val_dataset = make_testset(
dataroot=cfg.data.val_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 検証用データセット
test_dataset = make_testset(
dataroot=cfg.data.test_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 辞書型'train'と'val'と'test'のデータローダを作成
dataloaders_dict = create_dataloader(
batch_size=cfg.image.batch_size,
train_dataset=train_dataset,
val_dataset=val_dataset,
test_dataset=test_dataset,
)
# バッチごとの動作確認
batch_iterator = iter(dataloaders_dict["train"])
inputs, labels = next(batch_iterator)
print(inputs.size())
print(labels)
# ネットワークモデルのロード
net = models.vgg16_bn(pretrained=True)
log.info(net)
net.classifier[6] = nn.Linear(in_features=4096, out_features=2)
net.classifier[2] = nn.Dropout(p=0.6)
net.classifier[5] = nn.Dropout(p=0.6)
net.train()
# 損失関数の設定
criterion = nn.CrossEntropyLoss()
# criterion = nn.BCELoss()
log.info(net)
# 調整するパラメータの設定
params_to_update = []
update_param_names = cfg.train.update_param_names
# update_param_namesに含まれているパラメータだけ調整
for name, param in net.named_parameters():
if name in update_param_names:
param.requires_grad = True
params_to_update.append(param)
log.info(name)
else:
param.requires_grad = False
# 調整するパラメータ名をログに保存
log.info(params_to_update)
# 最適化手法の設定
optimizer = optim.SGD(params=params_to_update,
lr=cfg.optimizer.lr,
momentum=cfg.optimizer.momentum)
log.info(optimizer)
# 学習回数を設定ファイルから読み込む
num_epochs = cfg.train.num_epochs
# GPU初期設定
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("使用デバイス: ", device)
# ネットワークをGPUへ
net.to(device)
torch.backends.cudnn.benchmark = True
# 損失値と認識率を保持するリスト
train_loss = []
train_acc = []
test_loss = []
test_acc = []
# 学習と検証
for epoch in range(num_epochs):
log.info("Epoch {} / {}".format(epoch + 1, num_epochs))
log.info("----------")
# 学習
train_history = train_model(net, dataloaders_dict["train"], criterion,
optimizer)
# 学習したlossと認識率のリストを作成
train_loss.append(train_history["train_loss"])
train_acc.append(train_history["train_acc"])
# 検証
test_history = test_model(net, dataloaders_dict["val"], criterion)
# 検証したlossと認識率のリストを作成
test_loss.append(test_history["test_loss"])
test_acc.append(test_history["test_acc"])
# figインスタンスとaxインスタンスを作成
fig_loss, ax_loss = plt.subplots(figsize=(10, 10))
ax_loss.plot(range(1, num_epochs + 1, 1), train_loss, label="train_loss")
ax_loss.plot(range(1, num_epochs + 1, 1), test_loss, label="test_loss")
ax_loss.set_xlabel("epoch")
ax_loss.legend()
fig_loss.savefig("loss.png")
fig_acc, ax_acc = plt.subplots(figsize=(10, 10))
ax_acc.plot(range(1, num_epochs + 1, 1), train_acc, label="train_acc")
ax_acc.plot(range(1, num_epochs + 1, 1), test_acc, label="test_acc")
ax_acc.legend()
ax_acc.set_xlabel("epoch")
fig_acc.savefig("acc.png")
"""
# Pytorchのネットワークパラメータのロード
# 現在のディレクトリを取得
current_dir = pathlib.Path(__file__).resolve().parent
print(current_dir)
# 学習済みのパラメータを使用したいとき
load_path = str(current_dir) + "/weights_fine_tuning.pth"
load_weights = torch.load(load_path)
net.load_state_dict(load_weights)
"""
evaluate_history = evaluate_model(net, dataloaders_dict["test"], criterion)
print(evaluate_history["confusion_matrix"])
# 性能評価指標の計算（正解率、適合率、再現率、F1値)
efficienct = calculate_efficiency(evaluate_history["confusion_matrix"])
log.info("正解率: " + str(efficienct["accuracy"]))
log.info("適合率: " + str(efficienct["precision"]))
log.info("再現率: " + str(efficienct["recall"]))
log.info("f1値 :" + str(efficienct["f1"]))
# 混同行列の作成と表示
fig_conf, ax_conf = plt.subplots(figsize=(10, 10))
sns.heatmap(
evaluate_history["confusion_matrix"],
annot=True,
fmt="d",
cmap="Reds",
)
ax_conf.set_title("confusion_matrix")
ax_conf.set_xlabel("Predicted label")
ax_conf.set_ylabel("True label")
fig_conf.savefig("confusion_matrix.png")
# パラメータの保存
save_path = "./melanoma_nevi_classifier.pth"
torch.save(net.state_dict(), save_path)
"""
The main driver script
"""
from __future__ import print_function
import sys
from model import evaluate_model
if len(sys.argv) != 1:
print ("The correct syntax for running the script is python main.py")
evaluate_model()
def run_validation(data_path, n_times, num_epochs, train_df, valid_df):
for i in range(n_times):
print 'RUN {}'.format(i)
train_input = RNNInput(train_df)
config = RNNModelConfig.init_random()
print(config)
tf.reset_default_graph()
with tf.variable_scope("Model", reuse=None):
model = RNNModel(config)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Train model
start = time.time()
try:
train_losses = train_model(sess, model, train_input, num_epochs)
except ValueError:
train_loss = np.nan
valid_loss = np.nan
train_time = 0.0
valid_preds = []
else:
train_time = time.time() - start
# Dump figure
learning_fig = plt.semilogy(train_losses)[0].figure
fig_fname = '{}.png'.format(i)
learning_fig.savefig(os.path.join(data_path, fig_fname))
plt.close(learning_fig)
# Evaluate model
train_loss, _ = evaluate_model(sess, model, train_input)
valid_input = RNNInput(valid_df)
valid_loss, valid_preds = evaluate_model(sess, model, valid_input)
print "Training loss: {}".format(train_loss)
print
print "Validation loss: {}".format(valid_loss)
print
finally:
print 'Dumping...'
# Dump results
res = {
'train_loss': float(train_loss),
'train_time': float(train_time),
'valid_loss': float(valid_loss),
'config': config.as_dict(),
'valid_preds': np.array(valid_preds).astype(float).tolist()
}
res_fname = '{}.txt'.format(i)
with open(os.path.join(data_path, res_fname), 'w') as f:
json.dump(res, f)
print 'Done!'
print
if len(sys.argv) == 3:
stock_name, window_size = sys.argv[1], int(sys.argv[2])
elif len(sys.argv) == 4:
stock_name, window_size, model_name = sys.argv[1], int(
sys.argv[2]), sys.argv[3]
else:
print('Usage: python evaluate.py [stock] [window] [model (optional)]')
print(
'NOTE - All models in "models/" dir will be evaluated if no pretrained model is provided'
)
exit(0)
data = get_stock_data(stock_name)
initial_offset = data[1] - data[0]
if model_name is not None:
'''Single Model Evaluation'''
agent = Agent(window_size, pretrained=True, model_name=model_name)
profit = evaluate_model(agent, data, window_size=window_size)
show_eval_result(model_name, profit, initial_offset)
del agent
else:
'''Multiple Model Evaluation'''
for model in os.listdir('models'):
if not os.path.isdir('models/{}'.format(model)):
agent = Agent(window_size, pretrained=True, model_name=model)
profit = evaluate_model(agent, data, window_size=window_size)
show_eval_result(model, profit, initial_offset)
del agent
print('Done Evaluating!')
plt.gcf().savefig('loss.png')
if __name__ == '__main__':
# Load the dataset and split them into training and test sets
X_train, X_test, Y_train, Y_test = get_dataset()
# Create the model and compile it
model = create_model()
compile_model(model)
print(model.summary())
print()
print('Training model...')
training_history = fit_model(model, X_train, Y_train)
print()
print('Evaluating model...')
metrics = evaluate_model(model, X_test, Y_test)
print()
print('Loss on test set is:', metrics[0])
print('Accuracy on test set is:', metrics[-1])
print()
# Uncomment to see the plot of the training and validation losses (loss.png)
# print('Plotting training history...')
# plot_training_history(training_history)
# print('Done')
def main(filename):
print('loading data')
# Establish database connection
with open(
'/data/groups/schools1/mlpolicylab_fall20_schools1/pipeline/db_info.yaml',
'r') as f:
db_params = yaml.safe_load(f)['db']
engine = create_engine('postgres://:@{host}:{port}/{dbname}'.format(
host=db_params['host'],
port=db_params['port'],
dbname=db_params['dbname'],
))
# Load data from database to dataframe
df = load_data(filename, engine)
# Split dataframe into train and test data.
splits, years_reference = train_test_split(df)
for i, (train_df, test_df) in enumerate(splits):
print(f'processing split {i}')
# Explore data for each of the cohort
explore_data(train_df)
# Process train and test data seperately
updated_df_train = process_data(train_df)
updated_df_test = process_data(test_df)
# Upload the test and train data to database for future reference and easy retrival
updated_df_train.columns = [
col.replace('(', '').replace(')',
'').replace(' ',
'_').replace('/', '_')
for col in updated_df_train.columns
]
updated_df_test.columns = [
col.replace('(', '').replace(')',
'').replace(' ',
'_').replace('/', '_')
for col in updated_df_test.columns
]
table_name = timestamp + '_' + str(years_reference[i][1]) + '_' + str(
years_reference[i][0])
df_to_db(table_name, 'processed_data', updated_df_train,
updated_df_test, engine)
# Retreive test and train data from database
processed_train, processed_test = db_to_df(table_name,
'processed_data', engine)
updated_df_train_f = processed_train.copy()
updated_df_train_l = processed_train.copy()
updated_df_test_f = processed_test.copy()
updated_df_test_l = processed_test.copy()
# Create features for test and train data
features_train, train_student_ids = create_features(updated_df_train_f)
features_test, test_student_ids = create_features(updated_df_test_f)
# Create labels
label_train = create_label(updated_df_train_l)
label_test = create_label(updated_df_test_l)
# Concatenating features and labels to save in the database
train_concat = pd.concat([features_train, label_train],
axis=1,
sort=False)
test_concat = pd.concat([features_test, label_test],
axis=1,
sort=False)
# Calculating baseline rate using grade 9 gpa and base rate
baseline_precision = baseline(test_concat, years_reference[i])
base_rate = sum(train_concat.not_graduated) / len(train_concat)
# Saving and reading from database
df_to_db(table_name, 'model_data', train_concat, test_concat, engine)
model_train, model_test = db_to_df(table_name, 'model_data', engine)
features_train = model_train.iloc[:, :-1]
label_train = model_train.iloc[:, -1]
features_test = model_test.iloc[:, :-1]
label_test = model_test.iloc[:, -1]
# Build model
algos = ["Logistic", "SVM", "RandomForest", "DecisionTree"]
gs_params = {
"Logistic":
ParameterGrid({
'solver': ['lbfgs', 'liblinear', 'saga'],
'C': [0.001, 0.01, 0.1, 1, 2, 5, 10]
}),
"SVM":
ParameterGrid({
'C': [0.01, 1, 2, 5, 10],
'kernel': ['rbf', 'sigmoid']
}),
"RandomForest":
ParameterGrid({
'n_estimators': [30, 50, 100, 500, 1000, 10000],
'max_depth': [5, 10, 20, 50],
'min_samples_split': [5, 10, 15],
'max_features': ['auto', 'log2', 'sqrt']
}),
"DecisionTree":
ParameterGrid({
'criterion': ['gini', 'entropy'],
'max_depth': [5, 10, 20, 50],
'min_samples_split': [5, 10, 15]
})
}
print('performing model grid search')
for model_name in algos:
params = gs_params[model_name]
for param in params:
model = build_model(features_train, label_train, model_name,
param)
# Perform prediction
pred_proba_train = prediction(features_train, model)
pred_proba_test = prediction(features_test, model)
# Convert prediction probabilities to dataframes for further processing
pred_train_df = pd.DataFrame(pred_proba_train,
columns=['probability'])
pred_test_df = pd.DataFrame(pred_proba_test,
columns=['probability'])
# Retreive hyperparameters for processing
hyperparameters = ' '.join(
["{}: {}".format(key, param[key]) for key in param.keys()])
pred_train_df['model'], pred_train_df[
'params'] = model_name, hyperparameters
pred_test_df['model'], pred_test_df[
'params'] = model_name, hyperparameters
# Get the prediction scores for test and train data
predictions_train = pd.concat(
[train_student_ids, pred_train_df], axis=1, sort=False)
predictions_test = pd.concat([test_student_ids, pred_test_df],
axis=1,
sort=False)
# Calculate the bias metrics
TPR_gender, FDR_gender = bias_metrics(predictions_test,
processed_test, 'gender')
TPR_disadvantagement, FDR_disadvantagement = bias_metrics(
predictions_test, processed_test, 'disadvantagement')
# Load the prediction results to database for creating visualizations
df_to_db(table_name, 'predictions', predictions_train,
predictions_test, engine)
# Evaluate model
metric = evaluate_model(features_test,
label_test,
model,
model_name,
baseline_precision,
hyperparameters,
columns=model_train.columns[:-1])
# saving results
df_summary = pd.DataFrame({
'test_year':
years_reference[i][1],
'train_since':
years_reference[i][0],
'algorithm':
model_name,
'hyperparameters':
hyperparameters,
'baserate':
base_rate,
'baseline': [baseline_precision],
'precision':
metric,
'TPR_gender':
TPR_gender,
'FDR_gender':
FDR_gender,
'TPR_disadvantagement':
TPR_disadvantagement,
'FDR_disadvantagement':
FDR_disadvantagement
})
df_summary.to_sql(name=timestamp,
schema='performance_metrics',
con=engine,
if_exists='append',
index=False)
