def evaluate(logger, models, train_loaders, validation_loaders, test_loaders):
logger.info('')
logger.info(
'evaluating model on multiple sets combining both global-level and object-level models\' predictions'
)
logger.info('predict weights: ' + str(predict_weights[0]) + ', ' +
str(predict_weights[1]))
begin_time = time.time()
helper.evaluate(logger=logger,
models=models,
data_loaders=train_loaders,
set_name='train set',
predict_weights=predict_weights)
helper.evaluate(logger=logger,
models=models,
data_loaders=validation_loaders,
set_name='validation set',
predict_weights=predict_weights)
helper.evaluate(logger=logger,
models=models,
data_loaders=test_loaders,
set_name='test set',
predict_weights=predict_weights)
logger.info('evaluation has been done! total time: %.4fs' %
(time.time() - begin_time))
def test():
print("Loading test data...")
start_time = time.time()
x_test, y_test = process_file(test_dir, word_to_id, cat_to_id,
config.seq_length)
session = tf.Session()
session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess=session, save_path=save_path) # 读取保存的模型
print('Testing...')
loss_test, acc_test = evaluate(session, x_test, y_test)
msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}'
print(msg.format(loss_test, acc_test))
batch_size = 128
data_len = len(x_test)
num_batch = int((data_len - 1) / batch_size) + 1
y_test_cls = np.argmax(y_test, 1)
y_pred_cls = np.zeros(shape=len(x_test), dtype=np.int32) # 保存预测结果
for i in range(num_batch): # 逐批次处理
start_id = i * batch_size
end_id = min((i + 1) * batch_size, data_len)
feed_dict = {
model.input_x: x_test[start_id:end_id],
model.keep_prob: 1.0
}
y_pred_cls[start_id:end_id] = session.run(model.y_pred_cls,
feed_dict=feed_dict)
# 评估
print("Precision, Recall and F1-Score...")
print(
metrics.classification_report(y_test_cls,
y_pred_cls,
target_names=categories))
# 混淆矩阵
print("Confusion Matrix...")
cm = metrics.confusion_matrix(y_test_cls, y_pred_cls)
print(cm)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
session.close()
def binary_prediction(train, test, train_labels, test_labels, model):
"""
Runs and evaluates for the requested model for the binary classification.
:param train (pandas data frame): The training set of the data containing the features.
:param test (pandas data frame): The testing set of the data containing the features.
:param train_labels (pandas data frame): The training set of the data containing the labels.
:param test_labels (pandas data frame): The testing set of the data containing the labels.
:param model (string): The requested model to run.
:return: string containing the evaluation metrics values.
"""
t0 = time.time()
if model == 'SVM':
preds = predict_with_one_class_SVM(train, test, train_labels,
test_labels)
elif model == 'GNB':
preds = predict_with_GNB(train, test, train_labels, test_labels)
elif model == 'XGB':
train_labels = change_minority_class_label_to_zero(train_labels)
test_labels = change_minority_class_label_to_zero(test_labels)
preds = predict_with_XGB(train, test, train_labels, test_labels)
t1 = time.time()
if len(preds) > 0:
class_metrics = evaluate(test_labels, preds, n_decimals)
evaluation = print_evaluation(class_metrics)
runtime = str(round((t1 - t0) / secs, n_decimals))
return evaluation + runtime
def multi_prediction(train, test, train_labels, test_labels, model):
"""
Runs and evaluates for the requested model for the multi label classification.
:param train (pandas data frame): The training set of the data containing the features.
:param test (pandas data frame): The testing set of the data containing the features.
:param train_labels (pandas data frame): The training set of the data containing the labels.
:param test_labels (pandas data frame): The testing set of the data containing the labels.
:param model (string): The requested model to run.
:return: string containing the evaluation metrics values.
"""
t0 = time.time()
if model == 'SVM':
preds = predict_with_one_class_SVM_multi(train, test, train_labels,
test_labels)
elif model == 'GNB':
preds = predict_with_GNB_multi(train, test, train_labels, test_labels)
elif model == 'XGB':
train_labels = create_one_col_for_labels(train_labels)
test_labels = create_one_col_for_labels(test_labels)
preds = predict_with_XGB_multi(train, test, train_labels, test_labels)
t1 = time.time()
if len(preds) > 0:
class_metrics = evaluate(test_labels, preds, n_decimals, False)
evaluation = print_evaluation(class_metrics, False)
runtime = str(round((t1 - t0) / secs, n_decimals))
return evaluation + runtime
addLabel(models_graph)
plt.savefig('Bar_Charts_of_Models_and_their_Accuracy.png', dpi=300, transparent=True)
plt.show()
# ### Evaluating Models
# In[8]:
# Model Evaluation
for model_name, model in models:
helper.evaluate(X_test, y_test, model_name, model)
# ### Cross Validating Models
# #### Cross Validating with a single metric
# In[9]:
# Splitting data into 10 folds
cv_kfold = model_selection.KFold(n_splits=10, shuffle=True, random_state=23)
scorer = "r2"
model_names = []
cv_mean_scores = []
def main(_):
#initial outer file
WordIndex = Word_Index(FLAGS.word2id_path)
TarIndex = Tar_Tag_Index()
OpiIndex = Opi_Tag_Index()
FLAGS.num_word = len(WordIndex.word2idex)
FLAGS.num_class = TarIndex.num_class
df_test = pd.read_csv(FLAGS.test_file, sep='#', skip_blank_lines=False, dtype={'len': np.int32})
df_train = pd.read_csv(FLAGS.train_file, sep='#', skip_blank_lines=False, dtype={'len': np.int32})
df_train = df_train.iloc[np.random.permutation(len(df_train))].reset_index()
eval_size = int(len(df_train) * FLAGS.dev_rate)
df_eval = df_train.iloc[-eval_size:]
df_train = df_train.iloc[:-eval_size]
print('trainsize' + str(len(df_train)))
train_data_itor = DataItor(df_train)
eval_data_itor = DataItor(df_eval)
test_data_itor = DataItor(df_test)
FLAGS.check_every_point = int(train_data_itor.size / FLAGS.batch_size)
word2id, id2word = helper.loadMap(FLAGS.word2id_path)
if os.path.exists(FLAGS.pretrain_file):
FLAGS.pretrain_emb = initial_embedding_yelp_bin(word2id)
else:
FLAGS.pretrain_emb = None
myconfig = tf.ConfigProto(allow_soft_placement = True)
with tf.Session(config=myconfig) as sess:
model = BILSTM(FLAGS)
sess.run(tf.global_variables_initializer())
w_xs_eval, y_tuple_eval, couple_eval, lens_eval = eval_data_itor.next_all(FLAGS.use_couple)
w_xs_test, y_tuple_test, couple_test, lens_test = test_data_itor.next_all_no_padding(FLAGS.use_couple)
print('eval data size %f' % len(w_xs_eval))
# _, id2label = helper.loadMap(FLAGS.label2id_path)
saver = tf.train.Saver(max_to_keep=2)
previous_best_valid_f1_score = 0
previous_best_epoch = -1
bad_count = 0
heap_target, heap_opword = [], []
while train_data_itor.epoch < FLAGS.num_epochs:
x_train_batch, y_train_batch,couple_train_batch = train_data_itor.next_batch(FLAGS.batch_size,FLAGS.use_couple)
train_step, train_loss = model.train_model_union(sess, x_train_batch, y_train_batch,couples=couple_train_batch)
if train_data_itor.batch_time % FLAGS.check_every_point == 0:
print("current batch_time: %d" % (train_data_itor.batch_time))
opword_y_eval_pred, target_y_eval_pred, eval_loss, _,_ = model.inference_for_cpu(sess, w_xs_eval, y_tuple_eval, couple_eval)
# print('every loss:%f,%f' % (ent_loss, opi_loss))
precison, recall, target_f1_eval = helper.evaluate(w_xs_eval, y_tuple_eval[0],
target_y_eval_pred, id2word=id2word,seq_lens=lens_eval,
label_type='target')
print('evalution on eval data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % ( eval_loss, precison, recall, target_f1_eval))
opword_y_test_pred, target_y_test_pred, test_loss, all_ent_att_scores, all_opi_att_scores = model.inference_for_cpu(sess, w_xs_test, y_tuple_test,couple_test)
precison1, recall1, target_f1_test = helper.evaluate(w_xs_test, y_tuple_test[0],
target_y_test_pred, id2word=id2word,seq_lens=lens_test,
label_type='target')
print('evalution on test data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % ( test_loss, precison1, recall1, target_f1_test))
opword_precison, opword_recall, opword_f1_eval = helper.evaluate(w_xs_eval, y_tuple_eval[1],
opword_y_eval_pred, id2word=id2word,seq_lens=lens_eval,
label_type='opword')
print('evalution on eval data, opword_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % (eval_loss, opword_precison, opword_recall, opword_f1_eval))
# opword_y_test_pred, opword_test_loss = model.decode_opinion(sess, test_datas, y_opinions_test)
opword_precison, opword_recall, opword_f1_test = helper.evaluate(w_xs_test, y_tuple_test[1], opword_y_test_pred,
id2word=id2word,seq_lens=lens_test,
label_type='opword')
print('evalution on test data, opword_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % (test_loss, opword_precison, opword_recall, opword_f1_test))
if len(heap_target) < 5:
heapq.heappush(heap_target, (target_f1_eval, train_data_itor.epoch, target_f1_test,opword_f1_test))
else:
if target_f1_eval > heap_target[0][0]:
_, delete_file_epoch,_,_ = heapq.heappop(heap_target)
heapq.heappush(heap_target, (target_f1_eval, train_data_itor.epoch, target_f1_test,opword_f1_test))
if len(heap_opword) < 5:
heapq.heappush(heap_opword,(opword_f1_eval, train_data_itor.epoch, opword_f1_test))
else:
if opword_f1_eval > heap_opword[0][0]:
_, delete_file_epoch, _ = heapq.heappop(heap_opword)
heapq.heappush(heap_opword, (opword_f1_eval, train_data_itor.epoch, opword_f1_test))
# early stop
if target_f1_eval > previous_best_valid_f1_score:
previous_best_valid_f1_score = target_f1_eval
bad_count = 0
store_weights(all_ent_att_scores)
else:
bad_count += 1
if bad_count >= FLAGS.patients:
print('early stop!')
break
print('Train Finished!!')
# writer = codecs.open(exp_paths[task_id], 'a', 'utf-8')
# writer.close()
show_result(heap_target)
# show_result(heap_opword)
pass
gt_csv_path = 'crater_data/gt/gt_tile1_24.csv'
path1 = 'results/cnn/evaluations/' + removal_method1 + '/west_train_west_test_1_24_cnn_noduplicates.csv'
path2 = 'results/cnn/evaluations/' + removal_method2 + '/west_train_west_test_1_24_cnn_noduplicates.csv'
save_path = 'results/cnn/evaluations/NMS_BIRCH/west_train_west_test_1_24_cnn'
testset_name = 'tile1_24'
# the image for drawing rectangles
img_path = os.path.join('crater_data', 'images', testset_name + '.pgm')
gt_img = cv.imread(img_path)
gt = pd.read_csv(gt_csv_path, header=None)
no_dup_data1 = pd.read_csv(path1, header=None)
no_dup_data2 = pd.read_csv(path2, header=None)
start_time = time.time()
# compare the results of two duplicate removal methods
#evaluate_cmp(no_dup_data1, no_dup_data2, gt, gt_img, 64, True, save_path, param)
evaluate(gt, gt, gt_img, 64, True, save_path, param)
#img = draw_craters_rectangles(img_path, merge, show_probs=False)
#img = draw_craters_circles(img_path, merge, show_probs=False)
#cv.imwrite("%s.jpg" % (csv_path.split('.')[0]), img, [int(cv.IMWRITE_JPEG_QUALITY), 100])
end_time = time.time()
time_dif = end_time - start_time
print("Time usage: " + str(timedelta(seconds=int(round(time_dif)))))
resize_size=224, batch_size=32, object_boxes_dict=bounding_boxes)
logger.info('loading dataset costs %.4fs' % (time.time() - begin_time))
logger.info('loading models')
begin_time = time.time()
model_glb_name = 'resnet152'
model_glb = helper.get_model_by_name(model_glb_name, pretrained=False)
helper.replace_model_fc(model_glb_name, model_glb)
model_glb.load_state_dict(torch.load(model_glb_path))
model_obj_name = 'densenet161'
model_obj = helper.get_model_by_name(model_obj_name, pretrained=False)
helper.replace_model_fc(model_obj_name, model_obj)
model_obj.load_state_dict(torch.load(model_obj_path))
logger.info('loading models costs %.4fs' % (time.time() - begin_time))
models = [model_glb, model_obj]
validation_loaders = [valid_loader_glb, valid_loader_obj]
test_loaders = [test_loader_glb, test_loader_obj]
helper.evaluate(logger=logger,
models=models,
data_loaders=validation_loaders,
set_name='validation set',
predict_weights=predict_weights)
helper.evaluate(logger=logger,
models=models,
data_loaders=test_loaders,
set_name='test set',
predict_weights=predict_weights)
def train():
# 配置 Tensorboard，每次训练的结果保存在以日期时间命名的文件夹中。
print("Configuring TensorBoard and Saver...")
tensorboard_dir = 'tensorboard/textcnn' + '/' + time.strftime(
'%Y%m%d%H%M', time.localtime(time.time()))
if not os.path.exists(tensorboard_dir):
os.makedirs(tensorboard_dir)
tf.summary.scalar("loss", model.loss)
tf.summary.scalar("accuracy", model.acc)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(tensorboard_dir)
# 载入训练集与验证集
print("Loading training and validation data...")
start_time = time.time()
x_train, y_train, x_val, y_val = load_data(temp_dir, train_dir, val_dir,
word_to_id, cat_to_id,
config.seq_length)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
total_batch = tf.Variable(0, trainable=False) # 总批次，不可训练的变量
# 创建session
session = tf.Session()
# 导入权重
saver = load_model(session, save_dir)
# 图写入tensorboard
writer.add_graph(session.graph)
print('Training and evaluating...')
start_time = time.time()
best_acc_val = 0.0 # 最佳验证集准确率
last_improved = session.run(total_batch) # 记录上一次提升批次
require_improvement = 5000 # 如果超过1000轮未提升，提前结束训练
flag = False
for epoch in range(config.num_epochs):
print('Epoch:', epoch + 1)
batch_train = batch_iter(x_train, y_train, config.batch_size)
for x_batch, y_batch in batch_train:
feed_dict = feed_data(model, x_batch, y_batch,
config.dropout_keep_prob)
if session.run(total_batch) % config.save_per_batch == 0:
# 每多少轮次将训练结果写入tensorboard scalar
s = session.run(merged_summary, feed_dict=feed_dict)
writer.add_summary(s, session.run(total_batch))
if session.run(total_batch) % config.print_per_batch == 0:
# 每多少轮次输出在训练集和验证集上的性能
loss_train, F1_train, _, _ = evaluate(session, model, x_train,
y_train)
loss_val, F1_val, _, _ = evaluate(session, model, x_val,
y_val) # todo
if F1_val > best_acc_val:
# 保存最好结果
best_acc_val = F1_val
last_improved = session.run(total_batch)
saver.save(sess=session, save_path=save_path)
improved_str = '*'
else:
improved_str = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train F1: {2:>7.2%},' \
+ ' Val Loss: {3:>6.2}, Val F1: {4:>7.2%}, Time: {5} {6}'
print(
msg.format(session.run(total_batch), loss_train, F1_train,
loss_val, F1_val, time_dif, improved_str))
session.run(model.optim, feed_dict=feed_dict) # 运行优化
session.run(tf.assign(
total_batch, total_batch +
1)) # 用tf.assign迭代total_batch可以在saver中记录total_batch的变化
if session.run(total_batch) - last_improved > require_improvement:
# 验证集正确率长期不提升，提前结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break # 跳出循环
if flag: # 同上
break
session.close()
today_lst = [test_dct[stock][d] for stock in stock_lst]
total_value = np.array(shares).dot(np.array(today_lst))
print("Test day {}, cash is {:.2f}$, portfolio value is {:.2f}$".format(
d, cash, total_value + cash))
# print(shares)
print(actions)
res.append(total_value + cash)
np.save('./data/DQNAgent_vision.npy', np.array(res))
res = np.load('./data/DQNAgent_vision.npy', allow_pickle=True)
agent = np.load('./data/DQNAgent.npy', allow_pickle=True)
agent = agent[25:]
benchmark = np.array(benchmark)[1:]
benchmark = benchmark[25:]
rate_avg, sharpe = evaluate(res)
res = res[25:]
rate2 = (res[-1] - res[0]) / res[0]
rate3 = (benchmark[-1] - benchmark[0]) / benchmark[0]
print(
"total return {},rate of return avg {}, sharp ratio {}, benchmark_return {}"
.format(rate2, rate_avg, sharpe, rate3))
x = [i + 1 for i in range(len(res))]
sns.lineplot(x, benchmark / benchmark[0], label="SP500")
sns.lineplot(x, res / res[0], label="Agent_Vision")
sns.lineplot(x, agent / agent[0], label="Agent")
plt.xlabel("Days Testing")
plt.ylabel("Value (standardized)")
plt.legend()
plt.show()
output = model(Variable(input_tensor[seq_n]),
Variable(cap_tensor[seq_n]),
Variable(suf_tensor[seq_n]))
acc_output[:, seq_n, :] = output
for batch_i, batch_out in enumerate(acc_output):
best_score, best_path = model.viterbi_decode(batch_out)
for i, tag_i in enumerate(best_path):
word = words_input[i][args.window_size][batch_i]
real_tag = dataset.index_to_tag(target_tensor[batch_i][i])
predicted_tag = dataset.index_to_tag(tag_i)
tagged_file.write("{} {} {}\n".format(word, real_tag,
predicted_tag))
tagged_file.close()
acc, prec, rec, f1 = evaluate("tmp/dev_colo.annotated")
if f1 > dev_best_f1:
dev_best_f1 = f1
dev_best_epoch = epoch
# run test
tagged_file = open("tmp/test_colo.annotated", "w")
for i, (words_input, tags) in enumerate(test_loader):
input_tensor, target_tensor, cap_tensor, suf_tensor = model.get_train_tensors(
words_input,
tags,
word_to_index,
dataset.tag_to_index,
suffix2id=dataset.suffix_to_index)
acc_output = Variable(
v_type(1, len(input_tensor), dataset.get_num_tags()))
# loop sequence
# import random
from networkx import nx
from matplotlib import pyplot as plt
from helper import getNormalizedMatrix_np
from helper import evaluate
from networkx import nx
# In[L2]
allSC = np.load('pred0128/raw_sc_test_RESTING_L2.npy')
trueFC = np.load('pred0128/raw_fc_test_RESTING_L2.npy')
predFC = np.load(
'pred0128/prediction_SC_RESTINGSTATE_partialcorrelationFC_L2_set3_lm1000.0.npy'
)[:, :, :, 0]
# predFC = np.load('pred0128/raw_prediction_SC_RESTINGSTATE_partialcorrelationFC_L2_set3_lm1000.0.npy')[:,:,:,0]
pearson, diff = evaluate(predFC, trueFC, normalize=True)
print(np.mean(pearson), np.mean(diff))
# bestIdx = np.argmax(pearson)
minMSE = 100
for idx in range(allSC.shape[0]):
sc = getNormalizedMatrix_np(allSC[idx])
tfc = getNormalizedMatrix_np(trueFC[idx])
pfc = getNormalizedMatrix_np(predFC[idx])
# mse = np.linalg.norm(tfc-pfc)
# # mse = np.sum(np.abs(tfc-pfc))
# print(idx,mse)
# if mse<minMSE:
# sc_best = sc
# tfc_best = tfc
def main(_):
#initial outer file
WordIndex = Word_Index(FLAGS.word2id_path)
Index = [Tar_Tag_Index(), Opi_Tag_Index()]
object_tag_Index = Index[FLAGS.object_id]
FLAGS.dev_rate = 0.1
FLAGS.num_word = len(WordIndex.word2idex)
FLAGS.num_class = object_tag_Index.num_class
df_test = pd.read_csv(FLAGS.test_file,
sep='#',
skip_blank_lines=False,
dtype={'len': np.int32})
df_train = pd.read_csv(FLAGS.train_file,
sep='#',
skip_blank_lines=False,
dtype={'len': np.int32})
random_index = np.random.permutation(len(df_train))
df_train = df_train.iloc[random_index].reset_index()
print(random_index)
for i in range(len(random_index)):
print(random_index[i])
eval_size = int(len(df_train) * FLAGS.dev_rate)
df_eval = df_train.iloc[-eval_size:]
df_train = df_train.iloc[:-eval_size]
print('trainsize' + str(len(df_train)))
train_data_itor = DataItor(df_train)
eval_data_itor = DataItor(df_eval)
test_data_itor = DataItor(df_test)
FLAGS.check_every_point = int(train_data_itor.size / FLAGS.batch_size)
word2id, id2word = helper.loadMap(FLAGS.word2id_path)
if os.path.exists(FLAGS.pretrain_file):
FLAGS.pretrain_emb = initial_embedding_yelp_bin(word2id)
else:
FLAGS.pretrain_emb = None
myconfig = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=myconfig) as sess:
model = BILSTM(FLAGS)
sess.run(tf.global_variables_initializer())
w_xs_eval, y_tuple_eval, _, lens_eval = eval_data_itor.next_all(False)
w_xs_test, y_tuple_test, _, lens_test = test_data_itor.next_all_no_padding(
False)
print('eval data size %f' % len(w_xs_eval))
# _, id2label = helper.loadMap(FLAGS.label2id_path)
saver = tf.train.Saver(max_to_keep=2)
previous_best_valid_f1_score = 0
previous_best_epoch = -1
bad_count = 0
heap_target, heap_opword = [], []
while train_data_itor.epoch < FLAGS.num_epochs:
x_train_batch, y_train_batch, _ = train_data_itor.next_batch(
FLAGS.batch_size)
train_step, train_loss = model.train_model(
sess, x_train_batch, y_train_batch[FLAGS.object_id])
if train_data_itor.batch_time % FLAGS.check_every_point == 0:
print("current batch_time: %d" % (train_data_itor.batch_time))
y_eval_pred, eval_loss = model.inference_for_single(
sess, w_xs_eval, y_tuple_eval[FLAGS.object_id])
precison, recall, target_f1_eval = helper.evaluate(
w_xs_eval,
y_tuple_eval[FLAGS.object_id],
y_eval_pred,
id2word=id2word,
seq_lens=lens_eval,
label_type=types[FLAGS.object_id])
print(
'evalution on eval data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f'
% (eval_loss, precison, recall, target_f1_eval))
y_test_pred, test_loss = model.inference_for_single(
sess, w_xs_test, y_tuple_test[FLAGS.object_id])
precison1, recall1, target_f1_test = helper.evaluate(
w_xs_test,
y_tuple_test[FLAGS.object_id],
y_test_pred,
id2word=id2word,
seq_lens=lens_test,
label_type=types[FLAGS.object_id])
print(
'evalution on test data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f'
% (test_loss, precison1, recall1, target_f1_test))
if len(heap_target) < 5:
heapq.heappush(heap_target,
(target_f1_eval, train_data_itor.epoch,
target_f1_test))
else:
if target_f1_eval > heap_target[0][0]:
_, delete_file_epoch, _ = heapq.heappop(heap_target)
heapq.heappush(heap_target,
(target_f1_eval, train_data_itor.epoch,
target_f1_test))
# early stop
if target_f1_eval > previous_best_valid_f1_score:
previous_best_valid_f1_score = target_f1_eval
bad_count = 0
else:
bad_count += 1
if bad_count >= FLAGS.patients:
print('early stop!')
break
print('Train Finished!!')
show_result(heap_target)
pass
# In[40]:
model_parameters = pd.DataFrame(history.params)
model_parameters.to_csv("neural_network_parameters.csv", index=True)
model_parameters
# ### Model Evaluation
# In[16]:
model_name = 'Neural Network'
helper.evaluate(X_test, y_test, model_name, neural_network_model)
# ### Prediction
# In[17]:
y_pred = neural_network_model.predict(X_test)
# In[18]:
model_prediction_results = pd.DataFrame({
'actual_generation' : list(y_test),
# the image for drawing rectangles
img_path = os.path.join('crater_data', 'images', testset_name + '.pgm')
gt_img = cv.imread(img_path)
data = pd.read_csv(csv_path, header=None)
gt = pd.read_csv(gt_csv_path, header=None)
threshold = 0.75
start_time = time.time()
# first pass, remove duplicates for points of same window size
df1 = {}
merge = pd.DataFrame()
for ws in data[2].unique():
df1[ws] = data[ (data[3] > 0.75) & (data[2] == ws) ] # take only 75% or higher confidence
merge = pd.concat([merge, df1[ws]])
nodup = Banderia_duplicate_removal(merge)
# save the no duplicate csv file
nodup[[0,1,2]].to_csv("%s_noduplicates.csv" % save_path, header=False, index=False)
craters = nodup[[0,1,2]]
# evaluate with gt and draw it on final image.
dr, fr, qr, bf, f_measure, tp, fp, fn = evaluate(craters, gt, gt_img, 64, True, save_path, param)
end_time = time.time()
time_dif = end_time - start_time
print("Time usage: " + str(timedelta(seconds=int(round(time_dif)))))
def worker(gpu, ngpus_per_node, callback, args):
args.gpu = gpu
if args.distributed:
args.seed += args.gpu
torch.cuda.set_device(args.gpu)
args.rank = int(os.environ['RANK']) if 'RANK' in os.environ else 0
if args.multiprocessing_distributed:
args.rank = args.rank * ngpus_per_node + args.gpu
torch.distributed.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:8632',
world_size=args.world_size, rank=args.rank)
else:
args.rank = 0
if args.lr_scale:
scaled_lr = args.lr * math.sqrt((args.num_ales * args.world_size) / 16)
if args.rank == 0:
print('Scaled learning rate from {:4.4f} to {:4.4f}'.format(args.lr, scaled_lr))
args.lr = scaled_lr
args.use_cuda_env = args.use_cuda_env and torch.cuda.is_available()
args.no_cuda_train = (not args.no_cuda_train) and torch.cuda.is_available()
args.verbose = args.verbose and (args.rank == 0)
np.random.seed(args.seed)
torch.manual_seed(np.random.randint(1, 10000))
if args.use_cuda_env or (args.no_cuda_train == False):
torch.cuda.manual_seed(np.random.randint(1, 10000))
env_device = torch.device('cuda', args.gpu) if args.use_cuda_env else torch.device('cpu')
train_device = torch.device('cuda', args.gpu) if (args.no_cuda_train == False) else torch.device('cpu')
if args.rank == 0:
if args.output_filename:
train_csv_file = open(args.output_filename, 'w', newline='')
train_csv_writer = csv.writer(train_csv_file, delimiter=',')
train_csv_writer.writerow(['frames','fps','total_time',
'rmean','rmedian','rmin','rmax','rstd',
'lmean','lmedian','lmin','lmax','lstd',
'entropy','value_loss','policy_loss'])
eval_output_filename = '.'.join([''.join(args.output_filename.split('.')[:-1] + ['_test']), 'csv'])
eval_csv_file = open(eval_output_filename, 'w', newline='')
eval_csv_file.write(json.dumps(vars(args)))
eval_csv_file.write('\n')
eval_csv_writer = csv.writer(eval_csv_file, delimiter=',')
eval_csv_writer.writerow(['frames','total_time',
'rmean','rmedian','rmin','rmax','rstd',
'lmean','lmedian','lmin','lmax','lstd'])
else:
train_csv_file, train_csv_writer = None, None
eval_csv_file, eval_csv_writer = None, None
if args.plot:
from tensorboardX import SummaryWriter
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
log_dir = os.path.join(args.log_dir, current_time + '_' + socket.gethostname())
writer = SummaryWriter(log_dir=log_dir)
for k, v in vars(args).items():
writer.add_text(k, str(v))
print()
print('PyTorch : {}'.format(torch.__version__))
print('CUDA : {}'.format(torch.backends.cudnn.m.cuda))
print('CUDNN : {}'.format(torch.backends.cudnn.version()))
print('APEX : {}'.format('.'.join([str(i) for i in apex.amp.__version__.VERSION])))
print()
if train_device.type == 'cuda':
print(cuda_device_str(train_device.index), flush=True)
if args.use_openai:
train_env = create_vectorize_atari_env(args.env_name, args.seed, args.num_ales,
episode_life=args.episodic_life, clip_rewards=False,
max_frames=args.max_episode_length)
observation = torch.from_numpy(train_env.reset()).squeeze(1)
else:
train_env = AtariEnv(args.env_name, args.num_ales, color_mode='gray', repeat_prob=0.0,
device=env_device, rescale=True, episodic_life=args.episodic_life,
clip_rewards=False, frameskip=4)
train_env.train()
observation = train_env.reset(initial_steps=args.ale_start_steps, verbose=args.verbose).squeeze(-1)
if args.use_openai_test_env:
test_env = create_vectorize_atari_env(args.env_name, args.seed, args.evaluation_episodes,
episode_life=False, clip_rewards=False)
test_env.reset()
else:
test_env = AtariEnv(args.env_name, args.evaluation_episodes, color_mode='gray', repeat_prob=0.0,
device='cpu', rescale=True, episodic_life=False, clip_rewards=False, frameskip=4)
model = ActorCritic(args.num_stack, train_env.action_space, normalize=args.normalize, name=args.env_name)
model = model.to(train_device).train()
if args.rank == 0:
print(model)
args.model_name = model.name()
if args.use_adam:
optimizer = optim.Adam(model.parameters(), lr=args.lr, amsgrad=True)
else:
optimizer = optim.RMSprop(model.parameters(), lr=args.lr, eps=args.eps, alpha=args.alpha)
model, optimizer = amp.initialize(model, optimizer,
opt_level=args.opt_level,
loss_scale=args.loss_scale
)
if args.distributed:
model = DDP(model, delay_allreduce=True)
num_frames_per_iter = args.num_ales * args.num_steps
total_steps = math.ceil(args.t_max / (args.world_size * num_frames_per_iter))
shape = (args.num_steps + 1, args.num_ales, args.num_stack, *train_env.observation_space.shape[-2:])
states = torch.zeros(shape, device=train_device, dtype=torch.float32)
states[0, :, -1] = observation.to(device=train_device, dtype=torch.float32)
shape = (args.num_steps + 1, args.num_ales)
values = torch.zeros(shape, device=train_device, dtype=torch.float32)
returns = torch.zeros(shape, device=train_device, dtype=torch.float32)
shape = (args.num_steps, args.num_ales)
rewards = torch.zeros(shape, device=train_device, dtype=torch.float32)
masks = torch.zeros(shape, device=train_device, dtype=torch.float32)
actions = torch.zeros(shape, device=train_device, dtype=torch.long)
# These variables are used to compute average rewards for all processes.
episode_rewards = torch.zeros(args.num_ales, device=train_device, dtype=torch.float32)
final_rewards = torch.zeros(args.num_ales, device=train_device, dtype=torch.float32)
episode_lengths = torch.zeros(args.num_ales, device=train_device, dtype=torch.float32)
final_lengths = torch.zeros(args.num_ales, device=train_device, dtype=torch.float32)
if args.use_gae:
gae = torch.zeros(args.num_ales, device=train_device, dtype=torch.float32)
maybe_npy = lambda a: a.numpy() if args.use_openai else a
torch.cuda.synchronize()
iterator = range(total_steps)
if args.rank == 0:
iterator = tqdm(iterator)
total_time = 0
evaluation_offset = 0
for update in iterator:
T = args.world_size * update * num_frames_per_iter
if (args.rank == 0) and (T >= evaluation_offset):
evaluation_offset += args.evaluation_interval
eval_lengths, eval_rewards = evaluate(args, T, total_time, model, test_env, eval_csv_writer, eval_csv_file)
if args.plot:
writer.add_scalar('eval/rewards_mean', eval_rewards.mean().item(), T, walltime=total_time)
writer.add_scalar('eval/lengths_mean', eval_lengths.mean().item(), T, walltime=total_time)
start_time = time.time()
with torch.no_grad():
for step in range(args.num_steps):
value, logit = model(states[step])
# store values
values[step] = value.squeeze(-1)
# convert actions to numpy and perform next step
probs_action = F.softmax(logit, dim=1).multinomial(1).to(env_device)
observation, reward, done, info = train_env.step(maybe_npy(probs_action))
if args.use_openai:
# convert back to pytorch tensors
observation = torch.from_numpy(observation)
reward = torch.from_numpy(reward)
done = torch.from_numpy(done.astype(np.uint8))
else:
observation = observation.squeeze(-1).unsqueeze(1)
# move back to training memory
observation = observation.to(device=train_device)
reward = reward.to(device=train_device, dtype=torch.float32)
done = done.to(device=train_device)
probs_action = probs_action.to(device=train_device, dtype=torch.long)
not_done = 1.0 - done.float()
# update rewards and actions
actions[step].copy_(probs_action.view(-1))
masks[step].copy_(not_done)
rewards[step].copy_(reward.sign())
# update next observations
states[step + 1, :, :-1].copy_(states[step, :, 1:].clone())
states[step + 1] *= not_done.view(-1, *[1] * (observation.dim() - 1))
states[step + 1, :, -1].copy_(observation.view(-1, *states.size()[-2:]))
# update episodic reward counters
episode_rewards += reward
final_rewards[done] = episode_rewards[done]
episode_rewards *= not_done
episode_lengths += not_done
final_lengths[done] = episode_lengths[done]
episode_lengths *= not_done
returns[-1] = values[-1] = model(states[-1])[0].data.squeeze(-1)
if args.use_gae:
gae.zero_()
for step in reversed(range(args.num_steps)):
delta = rewards[step] + (args.gamma * values[step + 1] * masks[step]) - values[step]
gae = delta + (args.gamma * args.tau * masks[step] * gae)
returns[step] = gae + values[step]
else:
for step in reversed(range(args.num_steps)):
returns[step] = rewards[step] + (args.gamma * returns[step + 1] * masks[step])
value, logit = model(states[:-1].view(-1, *states.size()[-3:]))
log_probs = F.log_softmax(logit, dim=1)
probs = F.softmax(logit, dim=1)
action_log_probs = log_probs.gather(1, actions.view(-1).unsqueeze(-1))
dist_entropy = -(log_probs * probs).sum(-1).mean()
advantages = returns[:-1].view(-1).unsqueeze(-1) - value
value_loss = advantages.pow(2).mean()
policy_loss = -(advantages.clone().detach() * action_log_probs).mean()
loss = value_loss * args.value_loss_coef + policy_loss - dist_entropy * args.entropy_coef
optimizer.zero_grad()
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
optimizer.step()
states[0].copy_(states[-1])
torch.cuda.synchronize()
if args.rank == 0:
iter_time = time.time() - start_time
total_time += iter_time
if args.plot:
writer.add_scalar('train/rewards_mean', final_rewards.mean().item(), T, walltime=total_time)
writer.add_scalar('train/lengths_mean', final_lengths.mean().item(), T, walltime=total_time)
writer.add_scalar('train/learning_rate', scheduler.get_lr()[0], T, walltime=total_time)
writer.add_scalar('train/value_loss', value_loss, T, walltime=total_time)
writer.add_scalar('train/policy_loss', policy_loss, T, walltime=total_time)
writer.add_scalar('train/entropy', dist_entropy, T, walltime=total_time)
progress_data = callback(args, model, T, iter_time, final_rewards, final_lengths,
value_loss.item(), policy_loss.item(), dist_entropy.item(),
train_csv_writer, train_csv_file)
iterator.set_postfix_str(progress_data)
if args.plot:
writer.close()
if args.use_openai:
train_env.close()
if args.use_openai_test_env:
test_env.close()
param = Param.Param()
method_list = ["birch", "exp"]
#gt_list = ["1_24", "1_25", "2_24", "2_25", "3_24", "3_25"]
gt_list = ["1_24"]
for method in method_list:
print("evaluation of " + method + " approach")
for gt in gt_list:
# the image for drawing rectangles
print("working on tile" + gt)
tile_name = "tile" + gt
img_path = os.path.join('crater_data', 'tiles', tile_name + '.pgm')
gt_img = cv.imread(img_path)
gt_csv_path = os.path.join('crater_data', 'gt', gt + '_gt.csv')
gt_data = pd.read_csv(gt_csv_path, header=None)
# read detection from csv file.
dt_csv_path = os.path.join('results', 'crater-ception', method,
gt + '_sw_' + method + '.csv')
craters = pd.read_csv(dt_csv_path, header=None)
# save results path
save_path = 'results/crater-ception/' + method + '/evaluations/' + gt + '_sw_' + method
craters = craters[[0, 1, 2]]
# evaluate with gt and draw it on final image.
evaluate(craters, gt_data, gt_img, 64, True, save_path, param)
end_time = time.time()
time_dif = end_time - start_time
print("Time usage: " + str(timedelta(seconds=int(round(time_dif)))))
