def insert_table(base_time, instrument, con, table_type, count): if decideMarket(base_time): sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time) response = con.select_sql(sql) if len(response) == 0: if table_type == "1m": granularity = "M1" elif table_type == "5m": granularity = "M5" elif table_type == "1h": granularity = "H1" elif table_type == "3h": granularity = "H3" elif table_type == "8h": granularity = "H8" elif table_type == "day": granularity = "D" start_time = (base_time - timedelta(hours=14)).strftime("%Y-%m-%dT%H:%M:%S") params = { "from": start_time, "granularity": granularity, "price": "ABM", "count": count } req = instruments.InstrumentsCandles(instrument=instrument, params=params) client.request(req) response = req.response if len(response) == 0: pass else: for candle in response["candles"]: open_ask_price = candle["ask"]["o"] open_bid_price = candle["bid"]["o"] close_ask_price = candle["ask"]["c"] close_bid_price = candle["bid"]["c"] high_ask_price = candle["ask"]["h"] high_bid_price = candle["bid"]["h"] low_ask_price = candle["ask"]["l"] low_bid_price = candle["bid"]["l"] insert_time = candle["time"] insert_time = insert_time.split(".")[0] insert_time = insert_time + ".000000Z" #print(insert_time) insert_time = iso_jp(insert_time) insert_time = insert_time.strftime("%Y-%m-%d %H:%M:%S") sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, insert_time) response = con.select_sql(sql) if len(response) == 0: sql = "insert into %s_%s_TABLE(open_ask, open_bid, close_ask, close_bid, high_ask, high_bid, low_ask, low_bid, insert_time) values(%s, %s, %s, %s, %s, %s, %s, %s, \'%s\')" % (instrument, table_type, open_ask_price, open_bid_price, close_ask_price, close_bid_price, high_ask_price, high_bid_price, low_ask_price, low_bid_price, insert_time) print(sql) try: con.insert_sql(sql) except Exception as e: print(traceback.format_exc())
def check_table(base_time, instrument, con, table_type): if table_type == "1m": base_time = base_time - timedelta(minutes=1) elif table_type == "5m": base_time = base_time - timedelta(minutes=5) elif table_type == "1h": base_time = base_time - timedelta(hours=1) elif table_type == "3h": base_time = base_time - timedelta(hours=3) elif table_type == "8h": base_time = base_time - timedelta(hours=8) elif table_type == "day": base_time = base_time - timedelta(days=1) season = decideSeason(base_time) if season == "winter": base_time = base_time.strftime("%Y-%m-%d 07:00:00") base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S") else: base_time = base_time.strftime("%Y-%m-%d 06:00:00") base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S") if decideMarket(base_time): sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time) #print(sql) response = con.select_sql(sql) if len(response) == 0: print("below is *** ng *** %s" % sql) insert_table(base_time, instrument, con, table_type, count=1)
def run(self): for i in range(0, 100): # while True: base_time = self.getBaseTime() if self.old_base_time < base_time: if decideMarket(base_time): self.setPrice(base_time) self.setIndicator(base_time) self.old_base_time = base_time else: pass
def computeInsertIndicator(self, base_time, span): if decideMarket(base_time): self.setPrice(base_time) try: if span == "1h": self.set1hIndicator(base_time) elif span == "5m": self.set5mIndicator(base_time) elif span == "1m": self.set1mIndicator(base_time) else: pass except Exception as e: logging.info(traceback.format_exc()) else: pass
def getStartTime(self, base_time): hour = base_time.hour week = base_time.weekday() base_ftime = base_time.strftime("%Y-%m-%d 07:00:00") base_time = datetime.strptime(base_ftime, "%Y-%m-%d %H:%M:%S") if hour < 9: for i in range(1, 5): tmp_time = base_time - timedelta(days=i) if decideMarket(tmp_time): break base_time = tmp_time else: pass return base_time
def run(self): print "thread start" for i in range(0, 100): #print "thread start" # while True: base_time = self.getBaseTime() #print base_time #print "==========================" #print "base_time = %s" % base_time #print "old_base_time = %s" % self.old_base_time if self.old_base_time < base_time: print base_time if decideMarket(base_time): self.setPrice(base_time) self.setIndicator(base_time) self.old_base_time = base_time else: pass #print "thread end" print "thread end"
def bulk_insert(start_time, end_time, instrument, con, table_type): while start_time < end_time: if decideMarket(start_time): tmp_time = insert_table(start_time, instrument, con, table_type, count=5000) if type(tmp_time) == str: tmp_time = datetime.strptime(tmp_time, "%Y-%m-%d %H:%M:%S") if tmp_time <= start_time: if table_type == "5s": start_time = start_time + timedelta(seconds=5) elif table_type == "1m" or table_type == "5m" or table_type == "15m" or table_type == "30m": start_time = start_time + timedelta(minutes=1) elif table_type == "1h" or table_type == "3h" or table_type == "8h" or table_type == "day": start_time = start_time + timedelta(hours=1) else: start_time = tmp_time else: start_time = start_time + timedelta(seconds=1) else: start_time = start_time + timedelta(seconds=1)
def getHiLowPriceBeforeDay(self, base_time): before_day = base_time - timedelta(days=1) # 高値安値は直近1時間まで見てみる before_end_time = base_time - timedelta(hours=1) before_end_time = before_end_time.strftime("%Y-%m-%d %H:%M:%S") before_start_time = before_day.strftime("%Y-%m-%d 07:00:00") before_start_time = datetime.strptime(before_start_time, "%Y-%m-%d %H:%M:%S") if decideMarket(before_start_time): before_start_time = before_day.strftime("%Y-%m-%d 07:00:00") else: before_start_day = base_time - timedelta(days=3) before_start_time = before_start_day.strftime("%Y-%m-%d 07:00:00") sql = "select max(ask_price), max(bid_price) from %s_TABLE where insert_time > \'%s\' and insert_time < \'%s\'" % ( self.instrument, before_start_time, before_end_time) response = self.mysql_connector.select_sql(sql) for res in response: ask_price = res[0] bid_price = res[1] hi_price = (ask_price + bid_price) / 2 sql = "select min(ask_price), min(bid_price) from %s_TABLE where insert_time > \'%s\' and insert_time < \'%s\'" % ( self.instrument, before_start_time, before_end_time) response = self.mysql_connector.select_sql(sql) for res in response: ask_price = res[0] bid_price = res[1] min_price = (ask_price + bid_price) / 2 return hi_price, min_price
# ポーリング時間 sleep_time = 10 trade_wrapper = TradeWrapper(instrument, mode, test_mode, current_path, config_name, args, sendmail) trade_wrapper.setTradeAlgo(algo, base_time) try: while True: if test_mode: pass else: now = datetime.now() flag = decideMarket(base_time) if flag == False: sleep_time = 1 base_time = sleepTransaction(sleep_time, test_mode, base_time) elif test_mode == False and base_time > now: sleep_time = 1 base_time = sleepTransaction(sleep_time, test_mode, base_time) else: # 基本sleep_time = 0を返す sleep_time = trade_wrapper.setInstrumentResponse(base_time) base_time = sleepTransaction(sleep_time, test_mode, base_time) # order_flagがない時は、sleep_timeを返す
account_id = 4093685 token = 'e93bdc312be2c3e0a4a18f5718db237a-32ca3b9b94401fca447d4049ab046fad' env = 'live' mysql_connector = MysqlConnector() now = datetime.now() start_time = "2017-01-04 07:00:00" end_time = "2018-05-19 00:00:00" end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S") start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") sql_file = open("%s_record.sql" % instrument, "w") while start_time < end_time: if decideMarket(start_time): start_ftime = start_time - timedelta(hours=9) start_ftime = start_ftime.strftime("%Y-%m-%dT%H:%M:%S") oanda = oandapy.API(environment=env, access_token=token) response = {} try : response = oanda.get_history( instrument=instrument, start=start_ftime, granularity="S5" ) except ValueError as e: print(e) if len(response) > 0:
def train_save_model(window_size, output_train_index, table_type, figure_filename, model_filename, weights_filename, start_time, end_time, term): command = "ls ../model/ | grep -e %s -e %s | wc -l" % (model_filename, weights_filename) out = subprocess.getoutput(command) if int(out) < 2: start_ptime = change_to_ptime(start_time) end_ptime = change_to_ptime(end_time) target_time = start_ptime train_input_dataset = [] train_output_dataset = [] train_time_dataset = [] input_max_price = [] input_min_price = [] while target_time < end_ptime: hour = target_time.hour if decideMarket(target_time): if decideTerm(hour) == term or term == "all": if decideConditions(table_type, target_time): #if 1==1: print("term=%s, target_time=%s" % (term, target_time)) # 未来日付に変えて、教師データと一緒にまとめて取得 tmp_target_time = target_time - timedelta(hours=1) tmp_dataframe = get_original_dataset(target_time, table_type, span=window_size, direct="DESC") #tmp_output_dataframe = get_original_dataset(target_time, table_type, span=output_train_index, direct="ASC") tmp_output_dataframe = get_original_dataset_1h( target_time, table_type, span=output_train_index, direct="ASC") tmp_dataframe = pd.concat( [tmp_dataframe, tmp_output_dataframe]) tmp_time_dataframe = tmp_dataframe.copy( )["insert_time"] input_max_price.append(max(tmp_dataframe["end_price"])) input_min_price.append(min(tmp_dataframe["end_price"])) del tmp_dataframe["insert_time"] tmp_time_dataframe = pd.DataFrame(tmp_time_dataframe) tmp_time_input_dataframe = tmp_time_dataframe.iloc[: window_size, 0] tmp_time_output_dataframe = tmp_time_dataframe.iloc[-1, 0] #print("=========== train list ============") #print(tmp_time_input_dataframe) #print("=========== output list ============") #print(tmp_time_output_dataframe) tmp_np_dataset = tmp_dataframe.values normalization_model = build_to_normalization( tmp_np_dataset) tmp_np_normalization_dataset = change_to_normalization( normalization_model, tmp_np_dataset) tmp_dataframe = pd.DataFrame( tmp_np_normalization_dataset) tmp_input_dataframe = tmp_dataframe.copy( ).iloc[:window_size, :] tmp_output_dataframe = tmp_dataframe.copy().iloc[-1, 0] tmp_input_dataframe = tmp_input_dataframe.values train_time_dataset.append(tmp_time_output_dataframe) train_input_dataset.append(tmp_input_dataframe) train_output_dataset.append(tmp_output_dataframe) target_time = target_time + timedelta(hours=1) train_input_dataset = np.array(train_input_dataset) train_output_dataset = np.array(train_output_dataset) learning_model = build_learning_model(train_input_dataset, output_size=1, neurons=200) history = learning_model.fit(train_input_dataset, train_output_dataset, epochs=50, batch_size=1, verbose=2, shuffle=False) #history = learning_model.fit(train_input_dataset, train_output_dataset, epochs=1, batch_size=1, verbose=2, shuffle=False) train_predict = learning_model.predict(train_input_dataset) # 正規化戻しする paint_train_predict = [] paint_train_output = [] for i in range(len(input_max_price)): paint_train_predict.append( (train_predict[i][0] * (input_max_price[i] - input_min_price[i])) + input_min_price[i]) paint_train_output.append( (train_output_dataset[i] * (input_max_price[i] - input_min_price[i])) + input_min_price[i]) ### paint predict train data fig, ax1 = plt.subplots(1, 1) ax1.plot(train_time_dataset, paint_train_predict, label="Predict", color="blue") ax1.plot(train_time_dataset, paint_train_output, label="Actual", color="red") plt.savefig(figure_filename) # モデルの保存 model_filename = "../model/%s" % model_filename weights_filename = "../model/%s" % weights_filename json_string = learning_model.to_json() open(model_filename, "w").write(json_string) learning_model.save_weights(weights_filename) return learning_model
sys.path.append(current_path) sys.path.append(current_path + "/trade_algorithm") sys.path.append(current_path + "/obj") sys.path.append(current_path + "/lib") from mysql_connector import MysqlConnector from common import decideMarket from common import getSlope target_time = "2017-09-01 00:00:00" target_time = datetime.strptime(target_time, "%Y-%m-%d %H:%M:%S") mysqlConnector = MysqlConnector() output_file = open("regression_test.txt", "w") while True: if decideMarket(target_time): trend_time_width = 5 instrument = "GBP_JPY" sql = "select ask_price from %s_TABLE where insert_time <= \'%s\' and insert_time like \'%%59:59\' order by insert_time desc limit 10" % ( instrument, target_time) response = mysqlConnector.select_sql(sql) base_line_list = [] for res in response: base_line_list.append(res[0]) base_line_list.reverse() slope = getSlope(base_line_list) output_file.write("time = %s, slope = %s\n" % (target_time, slope))
def set1hIndicator(self, base_time): logging.info("set1hIndicator base_time = %s" % base_time) ask_price_list = self.indicator_object.getAskPriceList() bid_price_list = self.indicator_object.getBidPriceList() try: sql = "select insert_time from INDICATOR_TABLE where insert_time <= \'%s\' and type = \'bollinger1h3\' order by insert_time DESC limit 1" % base_time response = self.mysql_connector.select_sql(sql) polling_time = 3600 if self.calculatePollingTime(base_time, response, polling_time): # 1時間置きに実行 ind_type = "highlow" # 前日高値、安値の計算 if decideMarket(base_time - timedelta(hours=2)): high_price, low_price = self.getHiLowPrice(base_time) # instrument, type, high_price, low_price, insert_time sql = "insert into INDICATOR_TABLE(instrument, type, high_price, low_price, insert_time) values(\'%s\', \'%s\', %s, %s, \'%s\')" % ( self.instrument, ind_type, high_price, low_price, base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) ind_type = "ewma1h200" wma_length = 200 candle_width = 3600 # 移動平均の取得(WMA200 1h) ewma200_1h = getEWMA(ask_price_list, bid_price_list, wma_length, candle_width) # instrument, type, ewma_value, insert_time sql = "insert into INDICATOR_TABLE(instrument, type, ewma_value, insert_time) values(\'%s\', \'%s\', %s, \'%s\')" % ( self.instrument, ind_type, ewma200_1h[-1], base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) ind_type = "bollinger1h1" window_size = 28 candle_width = 3600 sigma_valiable = 1 data_set = getBollingerDataSet(ask_price_list, bid_price_list, window_size, sigma_valiable, candle_width) sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % ( self.instrument, ind_type, data_set["upper_sigmas"][-1], data_set["lower_sigmas"][-1], data_set["base_lines"][-1], base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) ind_type = "bollinger1h2" window_size = 28 candle_width = 3600 sigma_valiable = 2 data_set = getBollingerDataSet(ask_price_list, bid_price_list, window_size, sigma_valiable, candle_width) sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % ( self.instrument, ind_type, data_set["upper_sigmas"][-1], data_set["lower_sigmas"][-1], data_set["base_lines"][-1], base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) ind_type = "bollinger1h2.5" window_size = 28 candle_width = 3600 sigma_valiable = 2.5 data_set = getBollingerDataSet(ask_price_list, bid_price_list, window_size, sigma_valiable, candle_width) sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % ( self.instrument, ind_type, data_set["upper_sigmas"][-1], data_set["lower_sigmas"][-1], data_set["base_lines"][-1], base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) ind_type = "bollinger1h3" window_size = 28 candle_width = 3600 sigma_valiable = 3 data_set = getBollingerDataSet(ask_price_list, bid_price_list, window_size, sigma_valiable, candle_width) sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % ( self.instrument, ind_type, data_set["upper_sigmas"][-1], data_set["lower_sigmas"][-1], data_set["base_lines"][-1], base_time) self.mysql_connector.insert_sql(sql) logging.info(sql) except Exception as e: logging.info(traceback.format_exc())
def insert_table(base_time, instrument, con, table_type, count): debug_logger.info("%s: Start insert_table logic" % (base_time)) if decideMarket(base_time): sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
base_time = base_time - timedelta(hours=1) elif table_type == "3h": base_time = base_time - timedelta(hours=3) elif table_type == "8h": base_time = base_time - timedelta(hours=8) elif table_type == "day": base_time = base_time - timedelta(days=1) season = decideSeason(base_time) if season == "winter": base_time = base_time.strftime("%Y-%m-%d 07:00:00") base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S") else: base_time = base_time.strftime("%Y-%m-%d 06:00:00") base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S") if decideMarket(base_time): sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time) #print(sql) response = con.select_sql(sql) if len(response) == 0: print("below is *** ng *** %s" % sql) debug_logger.info("%s: No hit record = %s" % (base_time, sql)) insert_table(base_time, instrument, con, table_type, count=1) def insert_table(base_time, instrument, con, table_type, count): debug_logger.info("%s: Start insert_table logic" % (base_time)) if decideMarket(base_time): sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
token = account_data["token"] env = account_data["env"] if __name__ == "__main__": args = sys.argv currency = args[1].strip() con = MysqlConnector() polling_time = 0.5 sleep_time = 3600 units = 1000 oanda_wrapper = OandaWrapper(env, account_id, token, units) while True: try: now = datetime.now() flag = decideMarket(now) if flag == False: pass else: price_obj = oanda_wrapper.get_price(currency) ask_price = price_obj.getAskingPrice() bid_price = price_obj.getSellingPrice() sql = u"insert into %s_TABLE(ask_price, bid_price) values(%s, %s)" % (currency, ask_price, bid_price) con.insert_sql(sql) time.sleep(polling_time) except Exception as e: print e.args