parser.add_argument('--batch_size', default=128, type=int)
parser.add_argument('--epochs', default=2, type=int)
parser.add_argument('--gpu', default=0, type=int)
parser.add_argument('--kfold', default=5, type=int)
args = parser.parse_args()
'''
train_data = np.random.randint(0, high=config.MAX_WORDS, size=(10000, config.MAX_LENGTH))
train_label = np.random.randint(0, high=2, size=(10000, 6))
'''
# ctx = [mx.gpu(2), mx.gpu(3), mx.gpu(4), mx.gpu(5)]
ctx = utils.try_gpu()
net = net_define_eu()
# net.initialize(mx.init.Xavier(),ctx=ctx)
print("fetching train....")
train_data, train_label, word_index = fetch_data()
print("starting embedding...")
if not os.path.exists('../input/GD_EM.plk'):
embedding_dict = get_word_embedding()
print(len(word_index))
em = get_embed_matrix(embedding_dict, word_index)
with open('../input/GD_EM.plk', "wb") as f:
pickle.dump(em, f, -1)
else:
with open('../input/GD_EM.plk', "rb") as f:
em = pickle.load(f)
print('updating net...')
# print 'copy array'
em = array(em, ctx=mx.cpu())
def validation():
print('Begin validation')
final_predict_val = []
final_predict_index = []
groundtruth = []
with tf.Session() as sess:
graph = build_graph(top_k=3)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer()
) # initialize test_feeder's inside state
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print("restore from the checkpoint {0}".format(ckpt))
logger.info(':::Start validation:::')
try:
i = 0
acc_top_1, acc_top_k = 0.0, 0.0
while not coord.should_stop():
i += 1
start_time = time.time()
test_images_batch, test_labels_batch = fetch_data(
128, DIR_IMAGE_TRAIN)
feed_dict = {
graph['images']: test_images_batch,
graph['labels']: test_labels_batch,
graph['keep_prob']: 1.0,
graph['is_training']: False
}
batch_labels, probs, indices, acc_1, acc_k = sess.run(
[
graph['labels'], graph['predicted_val_top_k'],
graph['predicted_index_top_k'], graph['accuracy'],
graph['accuracy_top_k']
],
feed_dict=feed_dict)
final_predict_val += probs.tolist()
final_predict_index += indices.tolist()
groundtruth += batch_labels.tolist()
acc_top_1 += acc_1
acc_top_k += acc_k
end_time = time.time()
logger.info(
"the batch {0} takes {1} seconds, accuracy = {2}(top_1) {3}(top_k)"
.format(i, end_time - start_time, acc_1, acc_k))
except tf.errors.OutOfRangeError:
logger.info(
'==================Validation Finished================')
#
# acc_top_1 = acc_top_1 * FLAGS.batch_size / test_feeder.size
# acc_top_k = acc_top_k * FLAGS.batch_size / test_feeder.size
# logger.info('top 1 accuracy {0} top k accuracy {1}'.format(acc_top_1, acc_top_k))
finally:
coord.request_stop()
coord.join(threads)
return {
'prob': final_predict_val,
'indices': final_predict_index,
'groundtruth': groundtruth
}
def trade(context,bar_dict):
if context.trade_state.stop_sell == True:
if context.trade_state.sell_count >= 20:
context.trade_state.stop_sell = False
context.trade_state.sell_count = 0
else:
context.trade_state.sell_count += 1
if context.trade_state.stop_buy == True:
if context.trade_state.buy_count >= 20:
context.trade_state.stop_buy = False
context.trade_state.buy_count = 0
else:
context.trade_state.buy_count += 1
[hour,mins,sec] = bar_dict['time'].split(':')
if hour == '09' and mins == '01':
context.counter = 0
if context.last_main != context.main_futures[future_kinds_shanghai[0]]:
close_all(context, context.last_main)
context.counter += 1
if context.counter < 30:
return
look_back_num = 30
data = history_bars(future_kinds_shanghai[0]+'88',30, '1m', fields=['close'], skip_suspended=True)
data = [i[0] for i in data]
mean_price = np.average(data,weights=range(1,len(data)+1,1))
std_price = np.std(data)
std_price = sum(np.std(data)*range(1,len(data)+1,1))/sum(range(1,len(data)+1,1))
print(std_price)
is_closed = False
sell_num = context.future_account.positions[context.main_futures[future_kinds_shanghai[0]]].sell_quantity
buy_num = context.future_account.positions[context.main_futures[future_kinds_shanghai[0]]].buy_quantity
sell_pnl = context.avg_sell - bar_dict[context.main_futures[future_kinds_shanghai[0]]].close
buy_pnl = bar_dict[context.main_futures[future_kinds_shanghai[0]]].close - context.avg_buy
if sell_num !=0 or buy_num!=0:
print("sell_pnl:%f,sell_num:%d, buy_pnl:%f, buy_num:%d"%(sell_pnl,sell_num,buy_pnl,buy_num))
if (bar_dict[context.main_futures[future_kinds_shanghai[0]]].close > (mean_price + std_price)) and if_over and (context.stop_sell_open == False) and sell_num <= context.max_hold:
#print("close:%f, mean:%f , std:%f"%(bar_dict[context.main_futures[future_kinds_shanghai[0]]].close,mean_price,std_price))
data = fetch_data('sc','bu')
dates = data['datetime'].unique()
tick_size = 10
buy_multi = 2
sell_multi = 3
for date in dates:
day_data = data[data['datetime'] == date]
day_data = weighted_avg(day_data, name='weighted_avg', feature='last_price', if_include_now=False)
day_data['if_buy'] = day_data['bid_price'] > (day_data['weighted_avg'] + tick_size * buy_multi)
day_data['if_sell'] = day_data['ask_price'] < (day_data['weighted_avg'] - tick_size * sell_multi)
day_data['sell']
def train():
print('Begin training')
# ======================================================
# train_feeder = DataIterator(data_dir='../data/train/')
# test_feeder = DataIterator(data_dir='../data/test/')
# ======================================================
model_name = 'chinese-rec-model'
with tf.Session() as sess:
# ==================================================
# train_images, train_labels = train_feeder.input_pipeline(batch_size=FLAGS.batch_size, aug=True)
# test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size)
# ==================================================
graph = build_graph(top_k=1)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/val')
start_step = 0
if FLAGS.restore:
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt)
print("restore from the checkpoint {0}".format(ckpt))
start_step += int(ckpt.split('-')[-1])
logger.info(':::Training Start:::')
try:
i = 0
while not coord.should_stop():
i += 1
start_time = time.time()
# train_images_batch, train_labels_batch = sess.run([train_images, train_labels])
train_images_batch, train_labels_batch = fetch_data(
128, DIR_IMAGE_TRAIN)
feed_dict = {
graph['images']: train_images_batch,
graph['labels']: train_labels_batch,
graph['keep_prob']: 0.8,
graph['is_training']: True
}
_, loss_val, train_summary, step = sess.run(
[
graph['train_op'], graph['loss'],
graph['merged_summary_op'], graph['global_step']
],
feed_dict=feed_dict)
train_writer.add_summary(train_summary, step)
end_time = time.time()
logger.info("the step {0} takes {1} loss {2}".format(
step, end_time - start_time, loss_val))
if step > FLAGS.max_steps:
break
if step % FLAGS.eval_steps == 1:
test_images_batch, test_labels_batch = fetch_data(
128, DIR_IMAGE_TRAIN)
feed_dict = {
graph['images']: test_images_batch,
graph['labels']: test_labels_batch,
graph['keep_prob']: 1.0,
graph['is_training']: False
}
accuracy_test, test_summary = sess.run(
[graph['accuracy'], graph['merged_summary_op']],
feed_dict=feed_dict)
if step > 300:
test_writer.add_summary(test_summary, step)
logger.info(
'===============Eval a batch=======================')
logger.info('the step {0} test accuracy: {1}'.format(
step, accuracy_test))
logger.info(
'===============Eval a batch=======================')
if step % FLAGS.save_steps == 1:
logger.info('Save the ckpt of {0}'.format(step))
saver.save(sess,
os.path.join(FLAGS.checkpoint_dir, model_name),
global_step=graph['global_step'])
except tf.errors.OutOfRangeError:
logger.info('==================Train Finished================')
saver.save(sess,
os.path.join(FLAGS.checkpoint_dir, model_name),
global_step=graph['global_step'])
finally:
coord.request_stop()
coord.join(threads)
def get_batch_labels(batch_ab, encoder):
labels = []
for img in batch_ab:
label = encoder.soft_encode_with_color_rebal(img)
labels.append(label)
return np.asarray(labels)
# Ensure the checkpoint directory exists
if not os.path.exists("./checkpoints"):
os.makedirs("./checkpoints")
testing = False
model = IC_Model()
encoder = Encoder()
num_batches = int(floor(fetch_data() / hp.BATCH_SIZE))
num_batches = 3
# For saving/loading models
checkpoint_dir = './checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
checkpoint = tf.train.Checkpoint(model=model)
manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=3)
if not os.path.exists("./output"):
os.makedirs("./output")
if testing:
# restores the latest checkpoint using from the manager
checkpoint.restore(manager.latest_checkpoint).expect_partial()
test(model, encoder)
