parser.add_argument('--batch_size', default=128, type=int) parser.add_argument('--epochs', default=2, type=int) parser.add_argument('--gpu', default=0, type=int) parser.add_argument('--kfold', default=5, type=int) args = parser.parse_args() ''' train_data = np.random.randint(0, high=config.MAX_WORDS, size=(10000, config.MAX_LENGTH)) train_label = np.random.randint(0, high=2, size=(10000, 6)) ''' # ctx = [mx.gpu(2), mx.gpu(3), mx.gpu(4), mx.gpu(5)] ctx = utils.try_gpu() net = net_define_eu() # net.initialize(mx.init.Xavier(),ctx=ctx) print("fetching train....") train_data, train_label, word_index = fetch_data() print("starting embedding...") if not os.path.exists('../input/GD_EM.plk'): embedding_dict = get_word_embedding() print(len(word_index)) em = get_embed_matrix(embedding_dict, word_index) with open('../input/GD_EM.plk', "wb") as f: pickle.dump(em, f, -1) else: with open('../input/GD_EM.plk', "rb") as f: em = pickle.load(f) print('updating net...') # print 'copy array' em = array(em, ctx=mx.cpu())
def validation(): print('Begin validation') final_predict_val = [] final_predict_index = [] groundtruth = [] with tf.Session() as sess: graph = build_graph(top_k=3) saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer() ) # initialize test_feeder's inside state coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print("restore from the checkpoint {0}".format(ckpt)) logger.info(':::Start validation:::') try: i = 0 acc_top_1, acc_top_k = 0.0, 0.0 while not coord.should_stop(): i += 1 start_time = time.time() test_images_batch, test_labels_batch = fetch_data( 128, DIR_IMAGE_TRAIN) feed_dict = { graph['images']: test_images_batch, graph['labels']: test_labels_batch, graph['keep_prob']: 1.0, graph['is_training']: False } batch_labels, probs, indices, acc_1, acc_k = sess.run( [ graph['labels'], graph['predicted_val_top_k'], graph['predicted_index_top_k'], graph['accuracy'], graph['accuracy_top_k'] ], feed_dict=feed_dict) final_predict_val += probs.tolist() final_predict_index += indices.tolist() groundtruth += batch_labels.tolist() acc_top_1 += acc_1 acc_top_k += acc_k end_time = time.time() logger.info( "the batch {0} takes {1} seconds, accuracy = {2}(top_1) {3}(top_k)" .format(i, end_time - start_time, acc_1, acc_k)) except tf.errors.OutOfRangeError: logger.info( '==================Validation Finished================') # # acc_top_1 = acc_top_1 * FLAGS.batch_size / test_feeder.size # acc_top_k = acc_top_k * FLAGS.batch_size / test_feeder.size # logger.info('top 1 accuracy {0} top k accuracy {1}'.format(acc_top_1, acc_top_k)) finally: coord.request_stop() coord.join(threads) return { 'prob': final_predict_val, 'indices': final_predict_index, 'groundtruth': groundtruth }
def trade(context,bar_dict): if context.trade_state.stop_sell == True: if context.trade_state.sell_count >= 20: context.trade_state.stop_sell = False context.trade_state.sell_count = 0 else: context.trade_state.sell_count += 1 if context.trade_state.stop_buy == True: if context.trade_state.buy_count >= 20: context.trade_state.stop_buy = False context.trade_state.buy_count = 0 else: context.trade_state.buy_count += 1 [hour,mins,sec] = bar_dict['time'].split(':') if hour == '09' and mins == '01': context.counter = 0 if context.last_main != context.main_futures[future_kinds_shanghai[0]]: close_all(context, context.last_main) context.counter += 1 if context.counter < 30: return look_back_num = 30 data = history_bars(future_kinds_shanghai[0]+'88',30, '1m', fields=['close'], skip_suspended=True) data = [i[0] for i in data] mean_price = np.average(data,weights=range(1,len(data)+1,1)) std_price = np.std(data) std_price = sum(np.std(data)*range(1,len(data)+1,1))/sum(range(1,len(data)+1,1)) print(std_price) is_closed = False sell_num = context.future_account.positions[context.main_futures[future_kinds_shanghai[0]]].sell_quantity buy_num = context.future_account.positions[context.main_futures[future_kinds_shanghai[0]]].buy_quantity sell_pnl = context.avg_sell - bar_dict[context.main_futures[future_kinds_shanghai[0]]].close buy_pnl = bar_dict[context.main_futures[future_kinds_shanghai[0]]].close - context.avg_buy if sell_num !=0 or buy_num!=0: print("sell_pnl:%f,sell_num:%d, buy_pnl:%f, buy_num:%d"%(sell_pnl,sell_num,buy_pnl,buy_num)) if (bar_dict[context.main_futures[future_kinds_shanghai[0]]].close > (mean_price + std_price)) and if_over and (context.stop_sell_open == False) and sell_num <= context.max_hold: #print("close:%f, mean:%f , std:%f"%(bar_dict[context.main_futures[future_kinds_shanghai[0]]].close,mean_price,std_price)) data = fetch_data('sc','bu') dates = data['datetime'].unique() tick_size = 10 buy_multi = 2 sell_multi = 3 for date in dates: day_data = data[data['datetime'] == date] day_data = weighted_avg(day_data, name='weighted_avg', feature='last_price', if_include_now=False) day_data['if_buy'] = day_data['bid_price'] > (day_data['weighted_avg'] + tick_size * buy_multi) day_data['if_sell'] = day_data['ask_price'] < (day_data['weighted_avg'] - tick_size * sell_multi) day_data['sell']
def train(): print('Begin training') # ====================================================== # train_feeder = DataIterator(data_dir='../data/train/') # test_feeder = DataIterator(data_dir='../data/test/') # ====================================================== model_name = 'chinese-rec-model' with tf.Session() as sess: # ================================================== # train_images, train_labels = train_feeder.input_pipeline(batch_size=FLAGS.batch_size, aug=True) # test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size) # ================================================== graph = build_graph(top_k=1) saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/val') start_step = 0 if FLAGS.restore: ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) if ckpt: saver.restore(sess, ckpt) print("restore from the checkpoint {0}".format(ckpt)) start_step += int(ckpt.split('-')[-1]) logger.info(':::Training Start:::') try: i = 0 while not coord.should_stop(): i += 1 start_time = time.time() # train_images_batch, train_labels_batch = sess.run([train_images, train_labels]) train_images_batch, train_labels_batch = fetch_data( 128, DIR_IMAGE_TRAIN) feed_dict = { graph['images']: train_images_batch, graph['labels']: train_labels_batch, graph['keep_prob']: 0.8, graph['is_training']: True } _, loss_val, train_summary, step = sess.run( [ graph['train_op'], graph['loss'], graph['merged_summary_op'], graph['global_step'] ], feed_dict=feed_dict) train_writer.add_summary(train_summary, step) end_time = time.time() logger.info("the step {0} takes {1} loss {2}".format( step, end_time - start_time, loss_val)) if step > FLAGS.max_steps: break if step % FLAGS.eval_steps == 1: test_images_batch, test_labels_batch = fetch_data( 128, DIR_IMAGE_TRAIN) feed_dict = { graph['images']: test_images_batch, graph['labels']: test_labels_batch, graph['keep_prob']: 1.0, graph['is_training']: False } accuracy_test, test_summary = sess.run( [graph['accuracy'], graph['merged_summary_op']], feed_dict=feed_dict) if step > 300: test_writer.add_summary(test_summary, step) logger.info( '===============Eval a batch=======================') logger.info('the step {0} test accuracy: {1}'.format( step, accuracy_test)) logger.info( '===============Eval a batch=======================') if step % FLAGS.save_steps == 1: logger.info('Save the ckpt of {0}'.format(step)) saver.save(sess, os.path.join(FLAGS.checkpoint_dir, model_name), global_step=graph['global_step']) except tf.errors.OutOfRangeError: logger.info('==================Train Finished================') saver.save(sess, os.path.join(FLAGS.checkpoint_dir, model_name), global_step=graph['global_step']) finally: coord.request_stop() coord.join(threads)
def get_batch_labels(batch_ab, encoder): labels = [] for img in batch_ab: label = encoder.soft_encode_with_color_rebal(img) labels.append(label) return np.asarray(labels) # Ensure the checkpoint directory exists if not os.path.exists("./checkpoints"): os.makedirs("./checkpoints") testing = False model = IC_Model() encoder = Encoder() num_batches = int(floor(fetch_data() / hp.BATCH_SIZE)) num_batches = 3 # For saving/loading models checkpoint_dir = './checkpoints' checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") checkpoint = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=3) if not os.path.exists("./output"): os.makedirs("./output") if testing: # restores the latest checkpoint using from the manager checkpoint.restore(manager.latest_checkpoint).expect_partial() test(model, encoder)