def insert_table(base_time, instrument, con, table_type, count):
if decideMarket(base_time):
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
response = con.select_sql(sql)
if len(response) == 0:
if table_type == "1m":
granularity = "M1"
elif table_type == "5m":
granularity = "M5"
elif table_type == "1h":
granularity = "H1"
elif table_type == "3h":
granularity = "H3"
elif table_type == "8h":
granularity = "H8"
elif table_type == "day":
granularity = "D"
start_time = (base_time - timedelta(hours=14)).strftime("%Y-%m-%dT%H:%M:%S")
params = {
"from": start_time,
"granularity": granularity,
"price": "ABM",
"count": count
}
req = instruments.InstrumentsCandles(instrument=instrument, params=params)
client.request(req)
response = req.response
if len(response) == 0:
pass
else:
for candle in response["candles"]:
open_ask_price = candle["ask"]["o"]
open_bid_price = candle["bid"]["o"]
close_ask_price = candle["ask"]["c"]
close_bid_price = candle["bid"]["c"]
high_ask_price = candle["ask"]["h"]
high_bid_price = candle["bid"]["h"]
low_ask_price = candle["ask"]["l"]
low_bid_price = candle["bid"]["l"]
insert_time = candle["time"]
insert_time = insert_time.split(".")[0]
insert_time = insert_time + ".000000Z"
#print(insert_time)
insert_time = iso_jp(insert_time)
insert_time = insert_time.strftime("%Y-%m-%d %H:%M:%S")
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, insert_time)
response = con.select_sql(sql)
if len(response) == 0:
sql = "insert into %s_%s_TABLE(open_ask, open_bid, close_ask, close_bid, high_ask, high_bid, low_ask, low_bid, insert_time) values(%s, %s, %s, %s, %s, %s, %s, %s, \'%s\')" % (instrument, table_type, open_ask_price, open_bid_price, close_ask_price, close_bid_price, high_ask_price, high_bid_price, low_ask_price, low_bid_price, insert_time)
print(sql)
try:
con.insert_sql(sql)
except Exception as e:
print(traceback.format_exc())
def check_table(base_time, instrument, con, table_type):
if table_type == "1m":
base_time = base_time - timedelta(minutes=1)
elif table_type == "5m":
base_time = base_time - timedelta(minutes=5)
elif table_type == "1h":
base_time = base_time - timedelta(hours=1)
elif table_type == "3h":
base_time = base_time - timedelta(hours=3)
elif table_type == "8h":
base_time = base_time - timedelta(hours=8)
elif table_type == "day":
base_time = base_time - timedelta(days=1)
season = decideSeason(base_time)
if season == "winter":
base_time = base_time.strftime("%Y-%m-%d 07:00:00")
base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S")
else:
base_time = base_time.strftime("%Y-%m-%d 06:00:00")
base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S")
if decideMarket(base_time):
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
#print(sql)
response = con.select_sql(sql)
if len(response) == 0:
print("below is *** ng *** %s" % sql)
insert_table(base_time, instrument, con, table_type, count=1)
def run(self):
for i in range(0, 100):
# while True:
base_time = self.getBaseTime()
if self.old_base_time < base_time:
if decideMarket(base_time):
self.setPrice(base_time)
self.setIndicator(base_time)
self.old_base_time = base_time
else:
pass
def computeInsertIndicator(self, base_time, span):
if decideMarket(base_time):
self.setPrice(base_time)
try:
if span == "1h":
self.set1hIndicator(base_time)
elif span == "5m":
self.set5mIndicator(base_time)
elif span == "1m":
self.set1mIndicator(base_time)
else:
pass
except Exception as e:
logging.info(traceback.format_exc())
else:
pass
def getStartTime(self, base_time):
hour = base_time.hour
week = base_time.weekday()
base_ftime = base_time.strftime("%Y-%m-%d 07:00:00")
base_time = datetime.strptime(base_ftime, "%Y-%m-%d %H:%M:%S")
if hour < 9:
for i in range(1, 5):
tmp_time = base_time - timedelta(days=i)
if decideMarket(tmp_time):
break
base_time = tmp_time
else:
pass
return base_time
def run(self):
print "thread start"
for i in range(0, 100):
#print "thread start"
# while True:
base_time = self.getBaseTime()
#print base_time
#print "=========================="
#print "base_time = %s" % base_time
#print "old_base_time = %s" % self.old_base_time
if self.old_base_time < base_time:
print base_time
if decideMarket(base_time):
self.setPrice(base_time)
self.setIndicator(base_time)
self.old_base_time = base_time
else:
pass
#print "thread end"
print "thread end"
def bulk_insert(start_time, end_time, instrument, con, table_type):
while start_time < end_time:
if decideMarket(start_time):
tmp_time = insert_table(start_time,
instrument,
con,
table_type,
count=5000)
if type(tmp_time) == str:
tmp_time = datetime.strptime(tmp_time, "%Y-%m-%d %H:%M:%S")
if tmp_time <= start_time:
if table_type == "5s":
start_time = start_time + timedelta(seconds=5)
elif table_type == "1m" or table_type == "5m" or table_type == "15m" or table_type == "30m":
start_time = start_time + timedelta(minutes=1)
elif table_type == "1h" or table_type == "3h" or table_type == "8h" or table_type == "day":
start_time = start_time + timedelta(hours=1)
else:
start_time = tmp_time
else:
start_time = start_time + timedelta(seconds=1)
else:
start_time = start_time + timedelta(seconds=1)
def getHiLowPriceBeforeDay(self, base_time):
before_day = base_time - timedelta(days=1)
# 高値安値は直近1時間まで見てみる
before_end_time = base_time - timedelta(hours=1)
before_end_time = before_end_time.strftime("%Y-%m-%d %H:%M:%S")
before_start_time = before_day.strftime("%Y-%m-%d 07:00:00")
before_start_time = datetime.strptime(before_start_time,
"%Y-%m-%d %H:%M:%S")
if decideMarket(before_start_time):
before_start_time = before_day.strftime("%Y-%m-%d 07:00:00")
else:
before_start_day = base_time - timedelta(days=3)
before_start_time = before_start_day.strftime("%Y-%m-%d 07:00:00")
sql = "select max(ask_price), max(bid_price) from %s_TABLE where insert_time > \'%s\' and insert_time < \'%s\'" % (
self.instrument, before_start_time, before_end_time)
response = self.mysql_connector.select_sql(sql)
for res in response:
ask_price = res[0]
bid_price = res[1]
hi_price = (ask_price + bid_price) / 2
sql = "select min(ask_price), min(bid_price) from %s_TABLE where insert_time > \'%s\' and insert_time < \'%s\'" % (
self.instrument, before_start_time, before_end_time)
response = self.mysql_connector.select_sql(sql)
for res in response:
ask_price = res[0]
bid_price = res[1]
min_price = (ask_price + bid_price) / 2
return hi_price, min_price
# ポーリング時間
sleep_time = 10
trade_wrapper = TradeWrapper(instrument, mode, test_mode, current_path,
config_name, args, sendmail)
trade_wrapper.setTradeAlgo(algo, base_time)
try:
while True:
if test_mode:
pass
else:
now = datetime.now()
flag = decideMarket(base_time)
if flag == False:
sleep_time = 1
base_time = sleepTransaction(sleep_time, test_mode, base_time)
elif test_mode == False and base_time > now:
sleep_time = 1
base_time = sleepTransaction(sleep_time, test_mode, base_time)
else:
# 基本sleep_time = 0を返す
sleep_time = trade_wrapper.setInstrumentResponse(base_time)
base_time = sleepTransaction(sleep_time, test_mode, base_time)
# order_flagがない時は、sleep_timeを返す
account_id = 4093685
token = 'e93bdc312be2c3e0a4a18f5718db237a-32ca3b9b94401fca447d4049ab046fad'
env = 'live'
mysql_connector = MysqlConnector()
now = datetime.now()
start_time = "2017-01-04 07:00:00"
end_time = "2018-05-19 00:00:00"
end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S")
start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
sql_file = open("%s_record.sql" % instrument, "w")
while start_time < end_time:
if decideMarket(start_time):
start_ftime = start_time - timedelta(hours=9)
start_ftime = start_ftime.strftime("%Y-%m-%dT%H:%M:%S")
oanda = oandapy.API(environment=env, access_token=token)
response = {}
try :
response = oanda.get_history(
instrument=instrument,
start=start_ftime,
granularity="S5"
)
except ValueError as e:
print(e)
if len(response) > 0:
def train_save_model(window_size, output_train_index, table_type,
figure_filename, model_filename, weights_filename,
start_time, end_time, term):
command = "ls ../model/ | grep -e %s -e %s | wc -l" % (model_filename,
weights_filename)
out = subprocess.getoutput(command)
if int(out) < 2:
start_ptime = change_to_ptime(start_time)
end_ptime = change_to_ptime(end_time)
target_time = start_ptime
train_input_dataset = []
train_output_dataset = []
train_time_dataset = []
input_max_price = []
input_min_price = []
while target_time < end_ptime:
hour = target_time.hour
if decideMarket(target_time):
if decideTerm(hour) == term or term == "all":
if decideConditions(table_type, target_time):
#if 1==1:
print("term=%s, target_time=%s" % (term, target_time))
# 未来日付に変えて、教師データと一緒にまとめて取得
tmp_target_time = target_time - timedelta(hours=1)
tmp_dataframe = get_original_dataset(target_time,
table_type,
span=window_size,
direct="DESC")
#tmp_output_dataframe = get_original_dataset(target_time, table_type, span=output_train_index, direct="ASC")
tmp_output_dataframe = get_original_dataset_1h(
target_time,
table_type,
span=output_train_index,
direct="ASC")
tmp_dataframe = pd.concat(
[tmp_dataframe, tmp_output_dataframe])
tmp_time_dataframe = tmp_dataframe.copy(
)["insert_time"]
input_max_price.append(max(tmp_dataframe["end_price"]))
input_min_price.append(min(tmp_dataframe["end_price"]))
del tmp_dataframe["insert_time"]
tmp_time_dataframe = pd.DataFrame(tmp_time_dataframe)
tmp_time_input_dataframe = tmp_time_dataframe.iloc[:
window_size,
0]
tmp_time_output_dataframe = tmp_time_dataframe.iloc[-1,
0]
#print("=========== train list ============")
#print(tmp_time_input_dataframe)
#print("=========== output list ============")
#print(tmp_time_output_dataframe)
tmp_np_dataset = tmp_dataframe.values
normalization_model = build_to_normalization(
tmp_np_dataset)
tmp_np_normalization_dataset = change_to_normalization(
normalization_model, tmp_np_dataset)
tmp_dataframe = pd.DataFrame(
tmp_np_normalization_dataset)
tmp_input_dataframe = tmp_dataframe.copy(
).iloc[:window_size, :]
tmp_output_dataframe = tmp_dataframe.copy().iloc[-1, 0]
tmp_input_dataframe = tmp_input_dataframe.values
train_time_dataset.append(tmp_time_output_dataframe)
train_input_dataset.append(tmp_input_dataframe)
train_output_dataset.append(tmp_output_dataframe)
target_time = target_time + timedelta(hours=1)
train_input_dataset = np.array(train_input_dataset)
train_output_dataset = np.array(train_output_dataset)
learning_model = build_learning_model(train_input_dataset,
output_size=1,
neurons=200)
history = learning_model.fit(train_input_dataset,
train_output_dataset,
epochs=50,
batch_size=1,
verbose=2,
shuffle=False)
#history = learning_model.fit(train_input_dataset, train_output_dataset, epochs=1, batch_size=1, verbose=2, shuffle=False)
train_predict = learning_model.predict(train_input_dataset)
# 正規化戻しする
paint_train_predict = []
paint_train_output = []
for i in range(len(input_max_price)):
paint_train_predict.append(
(train_predict[i][0] *
(input_max_price[i] - input_min_price[i])) +
input_min_price[i])
paint_train_output.append(
(train_output_dataset[i] *
(input_max_price[i] - input_min_price[i])) +
input_min_price[i])
### paint predict train data
fig, ax1 = plt.subplots(1, 1)
ax1.plot(train_time_dataset,
paint_train_predict,
label="Predict",
color="blue")
ax1.plot(train_time_dataset,
paint_train_output,
label="Actual",
color="red")
plt.savefig(figure_filename)
# モデルの保存
model_filename = "../model/%s" % model_filename
weights_filename = "../model/%s" % weights_filename
json_string = learning_model.to_json()
open(model_filename, "w").write(json_string)
learning_model.save_weights(weights_filename)
return learning_model
sys.path.append(current_path)
sys.path.append(current_path + "/trade_algorithm")
sys.path.append(current_path + "/obj")
sys.path.append(current_path + "/lib")
from mysql_connector import MysqlConnector
from common import decideMarket
from common import getSlope
target_time = "2017-09-01 00:00:00"
target_time = datetime.strptime(target_time, "%Y-%m-%d %H:%M:%S")
mysqlConnector = MysqlConnector()
output_file = open("regression_test.txt", "w")
while True:
if decideMarket(target_time):
trend_time_width = 5
instrument = "GBP_JPY"
sql = "select ask_price from %s_TABLE where insert_time <= \'%s\' and insert_time like \'%%59:59\' order by insert_time desc limit 10" % (
instrument, target_time)
response = mysqlConnector.select_sql(sql)
base_line_list = []
for res in response:
base_line_list.append(res[0])
base_line_list.reverse()
slope = getSlope(base_line_list)
output_file.write("time = %s, slope = %s\n" % (target_time, slope))
def set1hIndicator(self, base_time):
logging.info("set1hIndicator base_time = %s" % base_time)
ask_price_list = self.indicator_object.getAskPriceList()
bid_price_list = self.indicator_object.getBidPriceList()
try:
sql = "select insert_time from INDICATOR_TABLE where insert_time <= \'%s\' and type = \'bollinger1h3\' order by insert_time DESC limit 1" % base_time
response = self.mysql_connector.select_sql(sql)
polling_time = 3600
if self.calculatePollingTime(base_time, response, polling_time):
# 1時間置きに実行
ind_type = "highlow"
# 前日高値、安値の計算
if decideMarket(base_time - timedelta(hours=2)):
high_price, low_price = self.getHiLowPrice(base_time)
# instrument, type, high_price, low_price, insert_time
sql = "insert into INDICATOR_TABLE(instrument, type, high_price, low_price, insert_time) values(\'%s\', \'%s\', %s, %s, \'%s\')" % (
self.instrument, ind_type, high_price, low_price,
base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
ind_type = "ewma1h200"
wma_length = 200
candle_width = 3600
# 移動平均の取得(WMA200 1h)
ewma200_1h = getEWMA(ask_price_list, bid_price_list,
wma_length, candle_width)
# instrument, type, ewma_value, insert_time
sql = "insert into INDICATOR_TABLE(instrument, type, ewma_value, insert_time) values(\'%s\', \'%s\', %s, \'%s\')" % (
self.instrument, ind_type, ewma200_1h[-1], base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
ind_type = "bollinger1h1"
window_size = 28
candle_width = 3600
sigma_valiable = 1
data_set = getBollingerDataSet(ask_price_list, bid_price_list,
window_size, sigma_valiable,
candle_width)
sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % (
self.instrument, ind_type, data_set["upper_sigmas"][-1],
data_set["lower_sigmas"][-1], data_set["base_lines"][-1],
base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
ind_type = "bollinger1h2"
window_size = 28
candle_width = 3600
sigma_valiable = 2
data_set = getBollingerDataSet(ask_price_list, bid_price_list,
window_size, sigma_valiable,
candle_width)
sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % (
self.instrument, ind_type, data_set["upper_sigmas"][-1],
data_set["lower_sigmas"][-1], data_set["base_lines"][-1],
base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
ind_type = "bollinger1h2.5"
window_size = 28
candle_width = 3600
sigma_valiable = 2.5
data_set = getBollingerDataSet(ask_price_list, bid_price_list,
window_size, sigma_valiable,
candle_width)
sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % (
self.instrument, ind_type, data_set["upper_sigmas"][-1],
data_set["lower_sigmas"][-1], data_set["base_lines"][-1],
base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
ind_type = "bollinger1h3"
window_size = 28
candle_width = 3600
sigma_valiable = 3
data_set = getBollingerDataSet(ask_price_list, bid_price_list,
window_size, sigma_valiable,
candle_width)
sql = "insert into INDICATOR_TABLE(instrument, type, upper_sigma, lower_sigma, base_line, insert_time) values(\'%s\', \'%s\', %s, %s, %s, \'%s\')" % (
self.instrument, ind_type, data_set["upper_sigmas"][-1],
data_set["lower_sigmas"][-1], data_set["base_lines"][-1],
base_time)
self.mysql_connector.insert_sql(sql)
logging.info(sql)
except Exception as e:
logging.info(traceback.format_exc())
def insert_table(base_time, instrument, con, table_type, count):
debug_logger.info("%s: Start insert_table logic" % (base_time))
if decideMarket(base_time):
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
base_time = base_time - timedelta(hours=1)
elif table_type == "3h":
base_time = base_time - timedelta(hours=3)
elif table_type == "8h":
base_time = base_time - timedelta(hours=8)
elif table_type == "day":
base_time = base_time - timedelta(days=1)
season = decideSeason(base_time)
if season == "winter":
base_time = base_time.strftime("%Y-%m-%d 07:00:00")
base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S")
else:
base_time = base_time.strftime("%Y-%m-%d 06:00:00")
base_time = datetime.strptime(base_time, "%Y-%m-%d %H:%M:%S")
if decideMarket(base_time):
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
#print(sql)
response = con.select_sql(sql)
if len(response) == 0:
print("below is *** ng *** %s" % sql)
debug_logger.info("%s: No hit record = %s" % (base_time, sql))
insert_table(base_time, instrument, con, table_type, count=1)
def insert_table(base_time, instrument, con, table_type, count):
debug_logger.info("%s: Start insert_table logic" % (base_time))
if decideMarket(base_time):
sql = "select insert_time from %s_%s_TABLE where insert_time = \'%s\'" % (instrument, table_type, base_time)
token = account_data["token"]
env = account_data["env"]
if __name__ == "__main__":
args = sys.argv
currency = args[1].strip()
con = MysqlConnector()
polling_time = 0.5
sleep_time = 3600
units = 1000
oanda_wrapper = OandaWrapper(env, account_id, token, units)
while True:
try:
now = datetime.now()
flag = decideMarket(now)
if flag == False:
pass
else:
price_obj = oanda_wrapper.get_price(currency)
ask_price = price_obj.getAskingPrice()
bid_price = price_obj.getSellingPrice()
sql = u"insert into %s_TABLE(ask_price, bid_price) values(%s, %s)" % (currency, ask_price, bid_price)
con.insert_sql(sql)
time.sleep(polling_time)
except Exception as e:
print e.args
