def main():
# load data
classes = open(DICT, 'r').read().splitlines()
nb_classes = len(classes)
train_samples = dtp.sample_paths_from_list(DATA_LOC, LIST_LOC+TRAIN_LIST)
eval_samples = dtp.sample_paths_from_list(DATA_LOC, LIST_LOC+EVAL_LIST)
sample = [train_samples[0]]
rgb_x, dep_x, y = dtp.load_data(sample, classes, DATA_LOC)
# load stream models
print 'Loading stream models...'
rgb_model = load_model(MODEL_LOC, RGB_MODEL_NAME)
dep_model = load_model(MODEL_LOC, DEP_MODEL_NAME)
# get fc6
rgb_func = theano.function([rgb_model.inputs['input_rgb'].get_input(train=False)],\
rgb_model.nodes['fc6_rgb'].get_output(train=False))
dep_func = theano.function([dep_model.inputs['input_dep'].get_input(train=False)],\
dep_model.nodes['fc6_dep'].get_output(train=False))
# delete models to save memory
del rgb_model
del dep_model
# create fusion model
fus_struct = create_fusion_model(nb_classes)
fus_model = train_model(fus_struct, rgb_func, dep_func, train_samples, eval_samples, classes)
save_model(fus_model, FUS_MODEL_NAME)
def load_data(input_file, **kargs):
"""
Memo
----
1. Example datasets
a. multivariate imputation applied
exposures-4yrs-merged-imputed.csv
b. rows with 'nan' dropped
exposures-4yrs-merged.csv
"""
import collections
import data_processor as dproc
X, y, features = dproc.load_data(input_path=dataDir, input_file=input_file, sep=',') # other params: input_path/None
print("(load_data) dim(X): {}, sample_size: {}".format(X.shape, X.shape[0]))
counts = collections.Counter(y)
print("... class distribution | classes: {} | sizes: {}".format(list(counts.keys()), counts))
print("... variables: {}".format(features))
return X, y, features
def predict(code, start, end):
test_data = cnn_data_generater.generate_predict_data(code, start, end)
model = cnn.load_model(model_path, code)
predicted = model.predict(test_data, verbose=1)
predicted = list(map(lambda n: n[0], predicted))
data = data_processor.load_data(data_path + str(code) + ".csv", start, end)
data = data_processor.divide(data, DATA_PERIOD, 0)
base_price = list(
map(lambda n: n[DATA_PERIOD - 1],
list(map(lambda n: n['CLOSE'].tolist(), data))))
date = list(
map(
lambda n: (datetime.strptime(n[DATA_PERIOD - 1], "%Y-%m-%d") +
timedelta(days=DATA_DURATION)).strftime("%Y-%m-%d"),
list(map(lambda n: n['DATE'].tolist(), data))))
df = pd.DataFrame({
'date': date,
'predict': predicted,
'base_price': base_price
})
df.to_csv("result.csv", index=False)
fig = (df.plot()).get_figure()
fig.savefig('../figure.png', dpi=600)
def test(code, start, end):
test_data = cnn_data_generater.generate_input_data(code, start, end)
model = cnn.load_model(model_path, code)
predicted = model.predict(test_data, verbose=1)
predicted = list(map(lambda n: n[0], predicted))
data = data_processor.load_data(data_path + str(code) + ".csv", start, end)
data = data_processor.divide(data, DATA_PERIOD + DATA_DURATION, 1)
base_price = list(
map(lambda n: n[DATA_PERIOD - 1],
list(map(lambda n: n['CLOSE'].tolist(), data))))
acual_price = list(
map(lambda n: n[DATA_PERIOD + DATA_DURATION - 1],
list(map(lambda n: n['CLOSE'].tolist(), data))))
date = list(
map(lambda n: n[DATA_PERIOD + DATA_DURATION - 1],
list(map(lambda n: n['DATE'].tolist(), data))))
df = pd.DataFrame({
'date': date,
'predict': predicted,
'base_price': base_price,
'acual_price': acual_price
})
df.to_csv("result.csv", index=False)
def train(args):
train_iter, dev_iter = data_processor.load_data(args) # 将数据分为训练集和验证集
print('加载数据完成')
model = TextRNN(args)
Cuda = torch.cuda.is_available()
if Cuda and args.cuda:
model.cuda()
"""
Q5:
Please give optimizer here
Add lr_scheduler to adjust learning rate.
"""
optimizer = torch.optim.Adam(model.parameters(), lr = args.lr)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.8)
steps = 0
best_acc = 0
last_step = 0
model.train()
for epoch in range(1, args.epoch + 1):
for batch in train_iter:
feature, target = batch.text, batch.label
# t_()函数表示将(max_len, batch_size)转置为(batch_size, max_len)
with torch.no_grad():
feature.t_(), target.sub_(1) # target减去1
if args.cuda and Cuda:
feature, target = feature.cuda(), target.cuda()
optimizer.zero_grad()
logits = model(feature)
loss = F.cross_entropy(logits, target)
loss.backward()
optimizer.step()
steps += 1
if steps % args.log_interval == 0:
# torch.max(logits, 1)函数：返回每一行中最大值的那个元素，且返回其索引（返回最大元素在这一行的列索引）
corrects = (torch.max(logits, 1)[1] == target).sum()
train_acc = 100.0 * corrects / batch.batch_size
sys.stdout.write(
'\rBatch[{}] - loss: {:.6f} acc: {:.4f}%({}/{})'.format(steps,
loss.item(),
train_acc,
corrects,
batch.batch_size))
if steps % args.test_interval == 0:
dev_acc = eval(dev_iter, model, args)
if dev_acc > best_acc:
best_acc = dev_acc
last_step = steps
if args.save_best:
print('Saving best model, acc: {:.4f}%\n'.format(best_acc))
save(model, args.save_dir, 'best', steps)
else:
scheduler.step()
print('lr decayed to {}'.format(optimizer.state_dict()['param_groups'][0]['lr']))
if steps - last_step >= args.early_stopping:
print('\nearly stop by {} steps, acc: {:.4f}%'.format(args.early_stopping, best_acc))
raise KeyboardInterrupt
def main(argv=None):
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list
# check if checkpoint path exists
if not os.path.exists(FLAGS.checkpoint_path):
os.mkdir(FLAGS.checkpoint_path)
else:
#if not FLAGS.restore:
# shutil.rmtree(FLAGS.checkpoint_path)
# os.mkdir(FLAGS.checkpoint_path)
shutil.rmtree(FLAGS.checkpoint_path)
os.mkdir(FLAGS.checkpoint_path)
train_data_generator = data_processor.generator(FLAGS)
train_samples_count = data_processor.count_samples(FLAGS)
val_data = data_processor.load_data(FLAGS)
if len(gpus) <= 1:
print('Training with 1 GPU')
if FLAGS.drn:
east = EAST_DRN_model(input_size=FLAGS.input_size)
else:
east = EAST_model(FLAGS.input_size)
parallel_model = east.model
else:
print('Training with %d GPUs' % len(gpus))
with tf.device("/cpu:0"):
east = EAST_model(FLAGS.input_size)
if FLAGS.restore_model is not '':
east.model.load_weights(FLAGS.restore_model)
parallel_model = multi_gpu_model(east.model, gpus=len(gpus))
score_map_loss_weight = K.variable(0.01, name='score_map_loss_weight')
small_text_weight = K.variable(0., name='small_text_weight')
lr_scheduler = LearningRateScheduler(lr_decay)
ckpt = CustomModelCheckpoint(model=east.model, path=FLAGS.checkpoint_path + '/model-{epoch:02d}.h5', period=FLAGS.save_checkpoint_epochs, save_weights_only=True)
tb = CustomTensorBoard(log_dir=FLAGS.checkpoint_path + '/train', score_map_loss_weight=score_map_loss_weight, small_text_weight=small_text_weight, data_generator=train_data_generator, write_graph=True)
small_text_weight_callback = SmallTextWeight(small_text_weight)
validation_evaluator = ValidationEvaluator(val_data, validation_log_dir=FLAGS.checkpoint_path + '/val')
callbacks = [lr_scheduler, ckpt, tb, small_text_weight_callback, validation_evaluator]
opt = AdamW(FLAGS.init_learning_rate)
parallel_model.compile(loss=[dice_loss(east.overly_small_text_region_training_mask, east.text_region_boundary_training_mask, score_map_loss_weight, small_text_weight),
rbox_loss(east.overly_small_text_region_training_mask, east.text_region_boundary_training_mask, small_text_weight, east.target_score_map)],
loss_weights=[1., 1.],
optimizer=opt)
east.model.summary()
model_json = east.model.to_json()
with open(FLAGS.checkpoint_path + '/model.json', 'w') as json_file:
json_file.write(model_json)
history = parallel_model.fit_generator(train_data_generator, epochs=FLAGS.max_epochs, steps_per_epoch=train_samples_count/FLAGS.batch_size, workers=FLAGS.nb_workers, use_multiprocessing=True, callbacks=callbacks, verbose=1)
def train(args):
train_iter, dev_iter = data_processor.load_data(args) # 将数据分为训练集和验证集
print('加载数据完成')
model = TextRNN(args)
if args.cuda: model.cuda()
"""
Q5:
Please give optimizer here
"""
optimizer = torch.optim.Adam(model.parameters())
steps = 0
best_acc = 0
last_step = 0
model.train()
for epoch in range(1, args.epoch + 1):
for batch in train_iter:
feature, target = batch.text, batch.label
# t_()函数表示将(max_len, batch_size)转置为(batch_size, max_len)
with torch.no_grad():
#feature.t_()
target.sub_(1) # target减去1
#print(feature.shape)
if args.cuda:
feature, target = feature.cuda(), target.cuda()
optimizer.zero_grad()
logits = model(feature)
#print(logits.shape)
loss = F.cross_entropy(logits, target)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
optimizer.step()
steps += 1
if steps % args.log_interval == 0:
# torch.max(logits, 1)函数：返回每一行中最大值的那个元素，且返回其索引（返回最大元素在这一行的列索引）
corrects = (torch.max(logits, 1)[1] == target).sum()
train_acc = 100.0 * corrects / batch.batch_size
sys.stdout.write(
'\rBatch[{}] - loss: {:.6f} acc: {:.4f}%({}/{})'.format(
steps, loss.item(), train_acc, corrects,
batch.batch_size))
if steps % args.test_interval == 0:
dev_acc = eval(dev_iter, model, args)
if dev_acc > best_acc:
best_acc = dev_acc
last_step = steps
if args.save_best:
print('Saving best model, acc: {:.4f}%\n'.format(
best_acc))
save(model, args.save_dir, 'best', steps)
else:
if steps - last_step >= args.early_stopping:
print('\nearly stop by {} steps, acc: {:.4f}%'.format(
args.early_stopping, best_acc))
raise KeyboardInterrupt
def generate_output_data(code, start, end):
data = data_processor.load_data(data_path + str(code) + ".csv", start, end)
data = data_processor.divide(data, DATA_PERIOD + DATA_DURATION, 1)
output_data = np.array(
list(
map(
lambda n: n[DATA_PERIOD + DATA_DURATION - 1] / n[DATA_PERIOD -
1],
list(map(lambda n: n['CLOSE'].tolist(),
data))))).reshape(len(data), 1)
return output_data
def make_prediction_matrix(test_samples, classes, func):
np.random.shuffle(test_samples)
test_samples = test_samples[:200]
for sample in test_samples:
rgb_x, dep_x, y = dtp.load_data([sample], classes, DATA_LOC)
out = func(rgb_x, dep_x)
if 'pred_mat' not in locals():
pred_mat = out
ground_truth = y
else:
pred_mat = np.concatenate((pred_mat, out), axis=0)
ground_truth = np.concatenate((ground_truth, y), axis=0)
return pred_mat, ground_truth
def load_data(cohort, **kargs):
"""
Memo
----
1. Example datasets
a. multivariate imputation applied
exposures-4yrs-merged-imputed.csv
b. rows with 'nan' dropped
exposures-4yrs-merged.csv
"""
import data_processor as dp
return dp.load_data(cohort, **kargs)
def generate_input_data(code, start, end):
data = data_processor.load_data(data_path + str(code) + ".csv", start, end)
data = data_processor.divide(data, DATA_PERIOD + DATA_DURATION, 1)
value_data = list(
map(lambda n: n.loc[:, ['CLOSE', 'OPEN', 'HIGH', 'LOW']].values, data))
value_data = list(map(lambda n: n[:DATA_PERIOD], value_data))
value_data = data_processor.normarize2D(np.array(value_data))
volume_data = list(map(lambda n: n['VOLUME'].tolist(), data))
volume_data = list(map(lambda n: data_processor.normarize(n), volume_data))
for i in range(len(value_data)):
for j in range(len(value_data[i])):
value_data[i][j].append(volume_data[i][j])
value_data = np.array(value_data).reshape(len(value_data), DATA_PERIOD, 5)
return value_data
def test_model(model, rgb_func, dep_func):
# load data
test_samples = dtp.sample_paths_from_list(DATA_LOC, LIST_LOC+TEST_LIST)
nb_test_samples = len(test_samples)
classes = open(DICT, 'r').read().splitlines()
nb_classes = len(classes)
# test
batch_size = 500
avg_loss = 0
avg_accuracy = 0
nb_test_batches = int(np.ceil(1.0*nb_test_samples/batch_size))
bar = progressbar.ProgressBar(maxval=nb_test_samples, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
for batch_id in range(0, nb_test_samples, batch_size):
bar.update(batch_id)
batch = test_samples[batch_id : batch_id+batch_size]
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
rgb_out = np.squeeze(rgb_func(rgb_x))
dep_out = np.squeeze(dep_func(dep_x))
fus_in = np.concatenate((rgb_out, dep_out), axis=1)
loss = model.test_on_batch({'input_fus': fus_in, 'output':y})
pred = model.predict({'input_fus': fus_in})
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
avg_loss += loss[0]
avg_accuracy += acc_count
bar.finish()
# show results
avg_loss /= nb_test_batches
avg_accuracy /= nb_test_samples
print 'Average loss: ' + str(avg_loss) + ' - Average accuracy: ' + str(avg_accuracy)
return avg_loss, avg_accuracy
def test_model(model):
# load data
test_samples = dtp.sample_paths_from_list(DATA_LOC, LIST_LOC+TEST_LIST)
nb_test_samples = len(test_samples)
classes = open(DICT, 'r').read().splitlines()
nb_classes = len(classes)
# test
batch_size = 30
avg_loss = 0
avg_accuracy = 0
nb_test_batches = int(np.ceil(1.0*nb_test_samples/batch_size))
bar = progressbar.ProgressBar(maxval=nb_test_samples, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
bar.start()
for batch_id in range(0, nb_test_samples, batch_size):
bar.update(batch_id)
batch = test_samples[batch_id : batch_id+batch_size]
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
params = {'input_rgb':rgb_x, 'input_dep':dep_x, 'output':y}
loss = model.test_on_batch(params)
del params['output']
pred = model.predict(params)
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
avg_loss += loss[0]
avg_accuracy += acc_count
bar.finish()
# show results
avg_loss /= nb_test_batches
avg_accuracy /= nb_test_samples
print 'Average loss: ' + str(avg_loss) + ' - Average accuracy: ' + str(avg_accuracy)
return avg_loss, avg_accuracy
def train_model(model, rgb_func, dep_func, train_samples, eval_samples, classes):
batch_size = 500
nb_epoch = 1000
patience = 5
nb_static_epoch = 0
epsilon = 1e-4
max_accuracy = 0
nb_train_samples = len(train_samples)
nb_eval_samples = len(eval_samples)
nb_train_batches = int(np.ceil(1.0*nb_train_samples/batch_size))
nb_eval_batches = int(np.ceil(1.0*nb_eval_samples/batch_size))
progress_widget = [progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]
train_bar = progressbar.ProgressBar(maxval=nb_train_samples, widgets=progress_widget)
eval_bar = progressbar.ProgressBar(maxval=nb_eval_samples, widgets=progress_widget)
f = open(LOG_LOC+'fus_fc6_train.log', 'w', 0)
for epoch in range(nb_epoch):
print 'Running epoch '+ str(epoch+1) + '/'+ str(nb_epoch) + '...'
f.write('Running epoch '+ str(epoch+1) + '/'+ str(nb_epoch) + '...\n')
# training------------------------------------------------------------------------------------
print 'Training phase'
f.write('Training phase--------------------------------------------------------------------\n')
train_bar.start()
start_time = time.time()
shuffle(train_samples)
for batch_id in range(0, nb_train_samples, batch_size):
train_bar.update(batch_id)
batch = train_samples[batch_id:batch_id+batch_size]
# create input
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
rgb_out = np.squeeze(rgb_func(rgb_x))
dep_out = np.squeeze(dep_func(dep_x))
fus_in = np.concatenate((rgb_out, dep_out), axis=1)
# train model
loss = model.train_on_batch({'input_fus': fus_in, 'output':y})
pred = model.predict({'input_fus': fus_in})
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
f.write(str(loss[0]) + ' --- ' + str(acc_on_batch) + '\n')
train_bar.finish()
elapsed_time = time.time() - start_time
print 'Elapsed time: ' + str(elapsed_time) + 's'
# evaluation------------------------------------------------------------------------------------
print 'Evaluation phase'
f.write('Evaluation phase--------------------------------------------------------------------\n')
avg_loss = 0
avg_accuracy = 0
eval_bar.start()
for batch_id in range(0, nb_eval_samples, batch_size):
eval_bar.update(batch_id)
batch = eval_samples[batch_id:batch_id+batch_size]
# create input
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
rgb_out = np.squeeze(rgb_func(rgb_x))
dep_out = np.squeeze(dep_func(dep_x))
fus_in = np.concatenate((rgb_out, dep_out), axis=1)
# test model
loss = model.test_on_batch({'input_fus': fus_in, 'output':y})
pred = model.predict({'input_fus': fus_in})
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
f.write(str(loss[0]) + ' --- ' + str(acc_on_batch) + '\n')
# accumulate loss and accuracy
avg_loss += loss[0]
avg_accuracy += acc_count
eval_bar.finish()
# check accuracy---------------------------------------------------------------------------------
avg_loss /= nb_eval_batches
avg_accuracy /= nb_eval_samples
improvement = avg_accuracy - max_accuracy
print 'Average loss: '+str(avg_loss)+' - Average accuracy: '+str(avg_accuracy)+' - Improvement: '+str(improvement)+'\n'
f.write('Average loss: '+str(avg_loss)+' - Average accuracy: '+str(avg_accuracy)+' - Improvement: '+str(improvement)+'\n\n')
if max_accuracy != 0:
improvement /= max_accuracy
if improvement > epsilon:
max_accuracy = avg_accuracy
nb_static_epoch = 0
else:
nb_static_epoch += 1
if nb_static_epoch >= patience:
print 'Accuracy does not imrpove. Early stopping...\n'
f.write('Accuracy does not imrpove. Early stopping...\n\n')
break
f.close()
return model
def main():
# load data
test_samples = dtp.sample_paths_from_list(DATA_LOC, LIST_LOC+TEST_LIST)
sample = [test_samples[0]]
classes = open(DICT, 'r').read().splitlines()
nb_classes = len(classes)
rgb_x, dep_x, y = dtp.load_data(sample, classes, DATA_LOC)
# load model
model = load_model(MODEL_LOC, FUS_MODEL_NAME)
# get node functions
node_func1 = theano.function([model.inputs['input_rgb'].get_input(train=False)],\
model.nodes['relu1_rgb'].get_output(train=False))
node_func2 = theano.function([model.inputs['input_rgb'].get_input(train=False)],\
model.nodes['relu5_rgb'].get_output(train=False))
node_func3 = theano.function([model.inputs['input_dep'].get_input(train=False)],\
model.nodes['relu1_dep'].get_output(train=False))
node_func4 = theano.function([model.inputs['input_dep'].get_input(train=False)],\
model.nodes['relu5_dep'].get_output(train=False))
node_func5 = theano.function([model.inputs['input_rgb'].get_input(train=False),\
model.inputs['input_dep'].get_input(train=False)],\
model.nodes['softmax_fus'].get_output(train=False) )
out1 = np.squeeze(node_func1(rgb_x))
out2 = np.squeeze(node_func2(rgb_x))
out3 = np.squeeze(node_func3(dep_x))
out4 = np.squeeze(node_func4(dep_x))
# visualize intermediate nodes
mosaic1 = make_mosaic(out1,10,10)
fig1 = plt.figure()
plt.imshow(mosaic1, cmap=cm.binary)
fig1.subtitle = 'relu1_rgb'
fig1.savefig('relu1_rgb.png')
mosaic2 = make_mosaic(out2,16,16)
fig2 = plt.figure()
plt.imshow(mosaic2, cmap=cm.binary)
fig2.subtitle = 'relu5_rgb'
fig2.savefig('relu5_rgb')
mosaic3 = make_mosaic(out3,10,10)
fig3 = plt.figure()
plt.imshow(mosaic3, cmap=cm.binary)
fig3.subtitle = 'relu1_dep'
fig3.savefig('relu1_dep.png')
mosaic4 = make_mosaic(out4,16,16)
fig4 = plt.figure()
plt.imshow(mosaic4, cmap=cm.binary)
fig4.subtitle = 'relu5_dep'
fig4.savefig('relu5_dep.png')
pred_mat, ground_truth = make_prediction_matrix(test_samples, classes, node_func5)
fig5 = plt.figure()
plt.subplot(1,2,1)
plt.imshow(pred_mat)
plt.subplot(1,2,2)
plt.imshow(ground_truth)
#fig5.subtitle('prediction')
fig5.savefig('prediction.png')
plt.show()
def train(args):
text = data_processor.load_data(args)
train_iter = data_processor.gen_batch(args, text)
dev_iter = data_processor.gen_test(text, args)
print('加载数据完成')
word_attn_model = AttentionWordGRU(args)
sent_attn_model = AttentionSentGRU(args)
model = None
if args.cuda: model.cuda()
word_optimizer = torch.optim.Adam(word_attn_model.parameters(), lr=args.lr)
sent_optimizer = torch.optim.Adam(sent_attn_model.parameters(), lr=args.lr)
steps = 0
best_acc = 0
last_step = 0
for epoch in range(1, args.epoch + 1):
for i in range(args.iterations):
word_optimizer.zero_grad()
sent_optimizer.zero_grad()
doc_texts, targets = next(train_iter)
# doc_texts的维度为(batch_size, sents_num, words_num)
word_attn_vectors = None
for doc_text in doc_texts:
# word_attn_vector的维度为(sent_num, hidden_size)
word_attn_vector = word_attn_model(doc_text)
# 将word_attn_vector的维度变为(1, sent_num, hidden_size)
word_attn_vector = word_attn_vector.unsqueeze(0)
if word_attn_vectors is None:
word_attn_vectors = word_attn_vector
else:
# word_attn_vectors的维度为(batch_size, sent_num, hidden_size)
word_attn_vectors = torch.cat(
(word_attn_vectors, word_attn_vector), 0)
logits = sent_attn_model(word_attn_vectors)
loss = F.cross_entropy(logits, targets)
loss.backward()
word_optimizer.step()
sent_optimizer.step()
steps += 1
if steps % args.log_interval == 0:
# torch.max(logits, 1)函数：返回每一行中最大值的那个元素，且返回其索引（返回最大元素在这一行的列索引）
corrects = (torch.max(logits, 1)[1] == targets).sum()
train_acc = 100.0 * corrects / args.batch_size
sys.stdout.write(
'\rBatch[{}] - loss: {:.6f} acc: {:.4f}%({}/{})'.format(
steps, loss.item(), train_acc, corrects,
args.batch_size))
if steps % args.test_interval == 0:
dev_acc = eval(dev_iter, word_attn_model, sent_attn_model)
if dev_acc > best_acc:
best_acc = dev_acc
last_step = steps
if args.save_best:
print('Saving best model, acc: {:.4f}%\n'.format(
best_acc))
save(word_attn_model, args.save_dir, 'best', steps)
save(sent_attn_model, args.save_dir, 'best', steps)
else:
if steps - last_step >= args.early_stopping:
print('\nearly stop by {} steps, acc: {:.4f}%'.format(
args.early_stopping, best_acc))
raise KeyboardInterrupt
import pandas as pd
import numpy as np
from keras.models import model_from_json
import lstm
import data_processor
code = 7203
start = "2014/01/01"
end = "2016/12/31"
data_path = "data/"
result_path = "result/"
DATA_PERIOD = 20
data = data_processor.load_data(data_path + str(code) + ".csv", start, end)
data = data_processor.divide(data_processor.rate(data['CLOSE'].tolist()),
DATA_PERIOD + 1)
test_data = np.array(list(map(lambda n: n[:DATA_PERIOD],
data))).reshape(len(data), DATA_PERIOD, 1)
print(test_data)
model = lstm.load_model(result_path, code)
predicted = model.predict(test_data, verbose=1)
print(predicted)
def main(argv=None):
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list
# check if checkpoint path exists
if not os.path.exists(FLAGS.checkpoint_path):
os.mkdir(FLAGS.checkpoint_path)
else:
#if not FLAGS.restore:
# shutil.rmtree(FLAGS.checkpoint_path)
# os.mkdir(FLAGS.checkpoint_path)
shutil.rmtree(FLAGS.checkpoint_path)
os.mkdir(FLAGS.checkpoint_path)
if len(gpus) <= 1:
print('Training with 1 GPU')
east = EAST_model(FLAGS.input_size)
parallel_model = east.model
else:
print('Training with %d GPUs' % len(gpus))
with tf.device("/cpu:0"):
east = EAST_model(FLAGS.input_size)
if FLAGS.restore_model is not '':
east.model.load_weights(FLAGS.restore_model)
parallel_model = multi_gpu_model(east.model, gpus=len(gpus))
score_map_loss_weight = K.variable(0.01, name='score_map_loss_weight')
small_text_weight = K.variable(0., name='small_text_weight')
# opt = AdamW(FLAGS.init_learning_rate)
opt = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-4)
parallel_model.compile(loss=[
dice_loss(east.overly_small_text_region_training_mask,
east.text_region_boundary_training_mask,
score_map_loss_weight, small_text_weight),
rbox_loss(east.overly_small_text_region_training_mask,
east.text_region_boundary_training_mask, small_text_weight,
east.target_score_map)
],
loss_weights=[1., 1.],
optimizer=opt)
east.model.summary()
model_json = east.model.to_json()
with open(FLAGS.checkpoint_path + '/model.json', 'w') as json_file:
json_file.write(model_json)
plot_model(east.model,
to_file=FLAGS.checkpoint_path + '/east-train.png',
show_shapes=True,
show_layer_names=True,
expand_nested=True)
plot_model(east.resnet,
to_file=FLAGS.checkpoint_path + '/resnet.png',
show_shapes=True,
show_layer_names=True,
expand_nested=True)
train_data_generator = data_processor.TrainDataSequence(FLAGS)
#train_data_generator = data_processor.generator(FLAGS)
#train_data_x, train_data_y = data_processor.load_train_data(FLAGS)
#print('# of train data: %d' %(len(train_data_x[0])))
train_samples_count = data_processor.count_samples(FLAGS)
print("train_samples_count: %d" % (train_samples_count))
val_data = data_processor.load_data(FLAGS)
lr_scheduler = LearningRateScheduler(lr_decay)
ckpt = CustomModelCheckpoint(model=east.model,
path=FLAGS.checkpoint_path,
period=FLAGS.save_checkpoint_epochs,
save_weights_only=False)
# tb = CustomTensorBoard(log_dir=FLAGS.checkpoint_path + '/train', score_map_loss_weight=score_map_loss_weight, small_text_weight=small_text_weight, data_generator=train_data_generator, write_graph=True)
small_text_weight_callback = SmallTextWeight(small_text_weight)
validation_evaluator = ValidationEvaluator(
val_data, validation_log_dir=FLAGS.checkpoint_path + '/val')
callbacks = [
lr_scheduler, ckpt, small_text_weight_callback, validation_evaluator
]
#callbacks = [lr_scheduler, ckpt, small_text_weight_callback]
#history = parallel_model.fit(x=train_data_x, y=train_data_y, batch_size=FLAGS.batch_size, epochs=FLAGS.max_epochs, verbose=1, callbacks=callbacks, max_queue_size=10, workers=FLAGS.nb_workers, use_multiprocessing=True)
#history = parallel_model.fit(x=train_data_generator, epochs=FLAGS.max_epochs, steps_per_epoch=train_samples_count/FLAGS.batch_size, callbacks=callbacks, max_queue_size=10, verbose=1)
history = parallel_model.fit(x=train_data_generator,
epochs=FLAGS.max_epochs,
callbacks=callbacks,
max_queue_size=10,
workers=FLAGS.nb_workers,
use_multiprocessing=False,
verbose=1)
file_name = FLAGS.checkpoint_path + '/model-train.h5'
east.model.save(file_name, overwrite=True)
plot_model(east.model_core,
to_file=FLAGS.checkpoint_path + '/east.png',
show_shapes=True,
show_layer_names=True,
expand_nested=True)
file_name = FLAGS.checkpoint_path + '/model.h5'
east.model_core.save(file_name, overwrite=True, include_optimizer=False)
tflite_model = lite.TFLiteConverter.from_keras_model_file(
file_name,
input_arrays=["input_image"],
input_shapes={
"input_image": [1, 224, 320, 3]
}).convert()
with open(file_name + '.tflite', 'wb') as tflite_file:
tflite_file.write(tflite_model)
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list
# check if checkpoint path exists
if not os.path.exists(FLAGS.checkpoint_path):
os.mkdir(FLAGS.checkpoint_path)
else:
# if not FLAGS.restore:
# shutil.rmtree(FLAGS.checkpoint_path)
# os.mkdir(FLAGS.checkpoint_path)
shutil.rmtree(FLAGS.checkpoint_path)
os.mkdir(FLAGS.checkpoint_path)
# train_data_generator = data_processor.generator(FLAGS)
train_data_generator = EastSequence(FLAGS, input_size=FLAGS.input_size)
train_samples_count = data_processor.count_samples(FLAGS)
val_data = data_processor.load_data(FLAGS)
train_graph = tf.Graph()
train_sess = tf.Session(graph=train_graph)
K.set_session(train_sess)
with train_graph.as_default():
K.set_learning_phase(0)
if len(gpus) <= 1:
print('Training with 1 GPU')
east = EASTModel(FLAGS.input_size)
parallel_model = east.model
else:
print('Training with %d GPUs' % len(gpus))
with tf.device("/cpu:0"):
east = EASTModel(FLAGS.input_size)
parallel_model = multi_gpu_model(east.model, gpus=len(gpus))
score_map_loss_weight = K.variable(0.01, name='score_map_loss_weight')
small_text_weight = K.variable(0., name='small_text_weight')
tf.contrib.quantize.create_training_graph(input_graph=train_graph,
quant_delay=250000)
train_sess.run(tf.global_variables_initializer())
lr_scheduler = LearningRateScheduler(lr_decay)
ckpt = CustomModelCheckpoint(model=east.model,
path=FLAGS.checkpoint_path +
'/model-{epoch:02d}.h5',
period=FLAGS.save_checkpoint_epochs,
save_weights_only=True)
tb = TensorBoard(log_dir=FLAGS.checkpoint_path + '/train',
batch_size=FLAGS.batch_size,
write_images=True)
small_text_weight_callback = SmallTextWeight(small_text_weight)
validation_evaluator = ValidationEvaluator(
val_data, validation_log_dir=FLAGS.checkpoint_path + '/val')
callbacks = [
lr_scheduler, ckpt, tb, small_text_weight_callback,
validation_evaluator
]
# opt = AdamW(FLAGS.init_learning_rate)
parallel_model.compile(loss=[
dice_loss(east.overly_small_text_region_training_mask,
east.text_region_boundary_training_mask,
score_map_loss_weight, small_text_weight),
rbox_loss(east.overly_small_text_region_training_mask,
east.text_region_boundary_training_mask,
small_text_weight, east.target_score_map)
],
loss_weights=[1., 1.],
optimizer='adam')
model_json = east.model.to_json()
with open(FLAGS.checkpoint_path + '/model.json', 'w') as json_file:
json_file.write(model_json)
history = parallel_model.fit_generator(
train_data_generator,
epochs=FLAGS.max_epochs,
steps_per_epoch=train_samples_count / FLAGS.batch_size,
workers=FLAGS.nb_workers,
use_multiprocessing=True,
max_queue_size=10,
callbacks=callbacks,
verbose=1)
saver = tf.train.Saver()
saver.save(train_sess, 'checkpoints')
tf.flags.DEFINE_string("checkpoint_dir", "cnn/model/checkpoints/",
"checkpoint directory")
tf.flags.DEFINE_integer("batch_size", 64, "Batch size")
# checkpoint_dir = sys.argv[1]
tf.flags.DEFINE_string("dev_dir", "Data/dev", "Data source for prediction")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
# print("\nParamenters:")
# for attribute, value in sorted(FLAGS.__flags.items()):
# print("{} = {}".format(attribute.upper(), value))
# print('\n')
#Load data
x_dev, y_real = data_processor.load_data(FLAGS.dev_dir)
num_class = y_real.shape[1]
#[[0,1][0,1][1,0][1,0]] ==> [1,1,1,1,0,0,0,0]
y_real = np.argmax(y_real, axis=1)
# print y_real
vocab_path = FLAGS.checkpoint_dir + "../vocab"
#加载与训练一样的分词器
vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
#分词与映射
x_map = np.array(list(vocab_processor.transform(x_dev)))
print("\nEvaluating ... \n")
#----------------------- Evaluation --------------------------------
#最新的model
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
tf.flags.DEFINE_integer("num_checkpoint", 5, "Number of saved checkpoint")
tf.flags.DEFINE_integer("evaluate_point", 100,
"Evaluate model on dev set after these steps")
tf.flags.DEFINE_integer("checkpoint", 100, "Save model after these steps")
# Parameters for data loading
tf.flags.DEFINE_float("n_folds", 10, "n_folds of cross_validation")
FLAGS = tf.flags.FLAGS #所有parameter
FLAGS._parse_flags()
#------------------- Load data and preprocess -------------------------
#Load
print("-Step 1: loading data...")
x_text, y_label = data_processor.load_data("Data/train")
"""
build vocabulary
whole_text: list: 一个sample是一个element
"""
#最长sample 长度
max_sample_length = 0
for text in x_text:
word_list = text.split(" ")
max_sample_length = max(max_sample_length, len(word_list))
# print max_sample_length
#high-level machine learning API (tf.contrib.learn)
#定义一个将所有文档映射成词索引的序列的实例，该序列最大为 max_sample_length
#使用分词器对文档进行分词，提取词袋，统计词被应用的次数,提取每一项文档的前 max_sample_length 个词的索引组合成新的X_train词袋的表现形式为：{"Abbott":1, "of":2, "Farnham":3, ....}
#定义分词器
def train_model(model, mode, batch_size, train_samples, eval_samples, classes):
'''
mode = 0: use only rgb
mode = 1: use only dep
mode = 2: use both rgb and dep
'''
nb_epoch = 1000
patience = 5
min_loss = sys.maxint
max_accuracy = 0
nb_static_epoch = 0
epsilon = 0.003
nb_train_samples = len(train_samples)
nb_eval_samples = len(eval_samples)
nb_train_batches = int(np.ceil(1.0*nb_train_samples/batch_size))
nb_eval_batches = int(np.ceil(1.0*nb_eval_samples/batch_size))
train_bar = progressbar.ProgressBar(maxval=nb_train_samples, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
eval_bar = progressbar.ProgressBar(maxval=nb_eval_samples, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
if mode==0:
f = open('rgb_train.log', 'w', 0)
elif mode==1:
f = open('dep_train.log', 'w', 0)
elif mode==2:
f = open('fus_train.log', 'w', 0)
for epoch in range(nb_epoch):
print 'Running epoch '+ str(epoch+1) + '/'+ str(nb_epoch) + '...'
f.write('Running epoch '+ str(epoch+1) + '/'+ str(nb_epoch) + '...\n')
# training---------------------------------------------------------
print 'Training phase'
f.write('Training phase------------------------------------------------\n')
train_bar.start()
start_time = time.time()
shuffle(train_samples)
for batch_id in range(0, nb_train_samples, batch_size):
train_bar.update(batch_id)
batch = train_samples[batch_id : batch_id+batch_size]
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
#x = switch_input(rgb_x, dep_x, mode)
#(loss, accuracy) = model.train_on_batch(x, y, accuracy=True)
params = gen_graph_input_params(rgb_x, dep_x, mode)
# find loss
params['output'] = y
loss = model.train_on_batch(params)
# find accuracy
del params['output']
pred = model.predict(params)
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
#pdb.set_trace()
f.write(str(loss[0]) + ' --- ' + str(acc_on_batch) + '\n')
train_bar.finish()
elapsed_time = time.time()-start_time
print 'Elapsed time: ' + str(elapsed_time) + 's'
# evaluating-------------------------------------------------------
print 'Evaluation phase'
f.write('Evaluation phase-----------------------------------------------\n')
avg_loss = 0
avg_accuracy = 0
eval_bar.start()
for batch_id in range(0, nb_eval_samples, batch_size):
eval_bar.update(batch_id)
batch = eval_samples[batch_id : batch_id+batch_size]
rgb_x, dep_x, y = dtp.load_data(batch, classes, DATA_LOC)
#x = switch_input(rgb_x, dep_x, mode)
#(loss, accuracy) = model.test_on_batch(x, y, accuracy=True)
params = gen_graph_input_params(rgb_x, dep_x, mode)
# find losss
params['output'] = y
loss = model.test_on_batch(params)
# find accuracy
del params['output']
pred = model.predict(params)
acc_count, acc_on_batch = compute_accuracy(pred.get('output'), y)
f.write(str(loss[0]) + ' --- ' + str(acc_on_batch) + '\n')
# accumulate loss and accuracy
avg_loss += loss[0]
avg_accuracy += acc_count
eval_bar.finish()
# check accuracy---------------------------------------------------
avg_loss /= nb_eval_batches
avg_accuracy /= nb_eval_samples
improvement = avg_accuracy - max_accuracy
print 'Average loss: '+str(avg_loss)+' - Average accuracy: '+str(avg_accuracy)+' - Improvement: '+str(improvement)+'\n'
f.write('Average loss: '+str(avg_loss)+' - Average accuracy: '+str(avg_accuracy)+' - Improvement: '+str(improvement)+'\n\n')
#print 'Accuracy: '+str(avg_accuracy)+' - Improvement: '+str(improvement)+'\n'
if max_accuracy != 0:
improvement /= max_accuracy
if improvement > epsilon:
#min_loss = avg_loss
max_accuracy = avg_accuracy
nb_static_epoch = 0
else:
nb_static_epoch += 1
if nb_static_epoch >= patience:
print 'Accuracy does not imrpove. Early stopping...\n'
f.write('Accuracy does not imrpove. Early stopping...\n\n')
break
f.close()
return model
def runWorkflow(**kargs):
def summarize_paths(paths, topk=10):
labels = np.unique(list(paths.keys()))
print("\n> 1. Frequent decision paths by labels (n={})".format(len(labels)))
sorted_paths = sort_path(paths, labels=labels, merge_labels=False, verbose=True)
for label in labels:
print("... Top {} paths (@label={}):\n{}\n".format(topk, label, sorted_paths[label][:topk]))
print("> 2. Frequent decision paths overall ...")
sorted_paths = sort_path(paths, labels=labels, merge_labels=True, verbose=True)
print("> Top {} paths (overall):\n{}\n".format(topk, sorted_paths[:topk]))
return
def summarize_vars(X, y):
counts = collections.Counter(y)
print("... class distribution | classes: {} | sizes: {}".format(list(counts.keys()), counts))
from data_processor import load_data
from utils_tree import visualize, sort_path
import operator
verbose = kargs.get('verbose', True)
# 1. define input dataset
if verbose: print("(runWorkflow) 1. Specifying input data ...")
######################################################
input_file = 'exposures-4yrs-merged-imputed.csv'
file_prefix = input_file.split('.')[0]
######################################################
X, y, features = load_data(input_path=dataDir, input_file=input_file, exclude_vars=['Gender', 'Zip Code'], verbose=True)
# 2. define model (e.g. decision tree)
if verbose: print("(runWorkflow) 2. Define model (e.g. decision tree and its parameters) ...")
######################################################
p_grid = {"max_depth": [3, 4, 5, 8, 10, 15],
"min_samples_leaf": [1, 5, 10, 15, 20]}
######################################################
# ... min_samples_leaf: the minimum number of samples required to be at a leaf node.
# A split point at any depth will only be considered if it leaves at least
# min_samples_leaf training samples in each of the left and right branches
model = DecisionTreeClassifier(criterion='entropy', random_state=1)
# 3. visualize the tree (deferred to analyze_path())
######################################################
test_size = 0.3
rs = 53
topk = 10
topk_vars = 10
######################################################
labels = [str(l) for l in sorted(np.unique(y))]
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=rs) # 70% training and 30% test
# params = {'max_depth': 5} # use model selection to determine the optimal 'max_depth'
# model = classify(X_train, y_train, params=params, random_state=rs)
# graph = visualize(model, features, labels=labels, plot_dir=plotDir, file_name=file_prefix, ext='tif')
# 4. analyze decision paths and keep track of frequent features
paths, counts = \
analyze_path(X, y, model=model, p_grid=p_grid, feature_set=features, n_trials=100, n_trials_ms=30, save=False,
merge_labels=False, policy_count='standard', experiment_id=file_prefix,
create_dir=True, index=0, validate_tree=False, to_str=True, verbose=False)
summarize_paths(paths, topk=topk)
# for k, ths in counts.items():
# assert isinstance(ths, list), "{} -> {}".format(k, ths)
fcounts = [(k, len(ths)) for k, ths in counts.items()]
sorted_features = sorted(fcounts, key=lambda x: x[1], reverse=True)
print("> Top {} features:\n{}\n".format(topk_vars, sorted_features[:topk_vars]))
return
