def get_orderbook_cursor(cls, mm1: str, mm2: str, coin_name: str,
start_time: int, end_time: int):
db_client = SharedMongoClient.instance()
mm1_col = db_client[mm1][coin_name + "_orderbook"]
mm2_col = db_client[mm2][coin_name + "_orderbook"]
return SharedMongoClient.get_data_from_db(mm1_col, mm2_col, start_time,
end_time)
def iyo_result_to_mongo_db(coin_name: str, start_time: int, end_time: int):
Global.configure_default_root_logging(should_log_to_file=False,
log_level=logging.CRITICAL)
SharedMongoClient.initialize(should_use_localhost_db=True)
db_client = SharedMongoClient.instance()
# convert epoch time to local_time and log
local_st = Global.convert_epoch_to_local_datetime(start_time,
timezone="kr")
local_et = Global.convert_epoch_to_local_datetime(end_time,
timezone="kr")
# create combination of coin that is injected by validating if the exchange has that coin
rfab_combi_list = Global.get_rfab_combination_tuples(coin_name)
for _combi in rfab_combi_list:
logging.critical(
"[%s-%s-%s] IYO conducting -> start_time: %s, end_time: %s" %
(coin_name.upper(), str(_combi[0]).upper(), str(
_combi[1]).upper(), local_st, local_et))
# draw iyo_config for bal & factor_setting
iyo_config = Global.read_iyo_setting_config(coin_name)
settings = TradeSettingConfig.get_settings(
mm1_name=_combi[0],
mm2_name=_combi[1],
target_currency=coin_name,
start_time=start_time,
end_time=end_time,
division=iyo_config["division"],
depth=iyo_config["depth"],
consecution_time=iyo_config["consecution_time"],
is_virtual_mm=True)
# todo
bal_factor_settings = TradeSettingConfig.get_bal_fact_settings(
iyo_config["krw_seq_end"], iyo_config["coin_seq_end"])
factor_settings = TradeSettingConfig.get_factor_settings(
_combi[0], _combi[1], coin_name,
iyo_config["max_trade_coin_end"], iyo_config["threshold_end"],
iyo_config["appx_unit_coin_price"])
try:
iyo_result = IntegratedYieldOptimizer.run(
settings, bal_factor_settings, factor_settings)
# finally save to mongoDB
if len(iyo_result) > 0:
db_client["statistics"]["iyo"].insert_many(iyo_result)
else:
logging.critical(
"There was no oppty!! Skipping to next combination!")
continue
except TypeError as e:
Global.send_to_slack_channel(
Global.SLACK_BOT_STATUS_URL,
"Something went wrong in IYO Schduler! >> %s" % e)
pass
def fill_empty_orderbook_entry(a_db: str, a_col: str, start_time: int,
end_time: int):
db_client = SharedMongoClient.instance()
a_target_col = db_client[a_db][a_col]
a_cursor = a_target_col.find({
"requestTime": {
"$gte": start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])
prev_item = None
for item in a_cursor:
if len(item["asks"]) == 0 or len(item["bids"]) == 0:
print(item["requestTime"])
if prev_item is None:
raise Exception("At least first item should not be None")
else:
item["asks"] = prev_item["asks"]
item["bids"] = prev_item["bids"]
SharedMongoClient._async_update(
a_target_col, {"requestTime": item["requestTime"]},
{"$set": item})
prev_item = item
def check_empty_data_by_rq_time(db_name: str, col_name: str,
start_time: int, end_time: int):
db_client = SharedMongoClient.instance()
target_col = db_client[db_name][col_name]
target_data_set = target_col.find({
"requestTime": {
"$gte": start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])
pre_data = None
for data in target_data_set:
if pre_data is None:
pre_data = data
continue
rq_diff = (data["requestTime"] - pre_data["requestTime"])
if rq_diff <= 7:
pre_data = data
else:
logging.info(
"RequestTime Difference observed! requestTime_diff: %d Current: %d, Before: %d"
% (rq_diff, data["requestTime"], pre_data["requestTime"]))
pre_data = data
import logging
from config.global_conf import Global
from config.shared_mongo_client import SharedMongoClient
from optimizer.integrated_yield_optimizer import IYOStatAppender
Global.configure_default_root_logging(should_log_to_file=False, log_level=logging.INFO)
SharedMongoClient.initialize(should_use_localhost_db=True)
start_time = Global.convert_local_datetime_to_epoch("2018.07.24 11:00:00", timezone="kr")
end_time = Global.convert_local_datetime_to_epoch("2018.07.24 12:00:00", timezone="kr")
db_client = SharedMongoClient.instance()
iyo_col = db_client["statistics"]["iyo"]
iyo_cur = iyo_col.find({
"settings.start_time": {
"$gte": start_time,
"$lte": end_time}}).sort([("start_time", 1)])
for iyo_data in iyo_cur:
logging.critical("Now starting: %s" % Global.convert_epoch_to_local_datetime(iyo_data["settings"]["start_time"]))
stat_result = IYOStatAppender.run(iyo_data=iyo_data)
# append this stat dict to the current IYO_data
db_client["statistics"]["iyo"].update(
{'_id': iyo_data['_id']}, {
'$set': {
'stat': stat_result
}}, upsert=False, multi=False)
logging.critical("Appended Stat result to target IYO_data at MongoDB")
def main(coin_name: str, init_time: str, final_time: str):
Global.configure_default_root_logging(should_log_to_file=False,
log_level=logging.INFO)
SharedMongoClient.initialize(should_use_localhost_db=False)
db_client = SharedMongoClient.instance()
time_list = make_time_list(init_time, final_time)
prev_time = None
for cur_time in time_list:
if prev_time is None:
prev_time = cur_time
continue
logging.warning("Nohup conducting -> start_time: %s, end_time: %s" %
(prev_time, cur_time))
# Global.send_to_slack_channel("IYO Initiated!! start_time: %s, end_time: %s" % (prev_time, cur_time))
start_time = Global.convert_local_datetime_to_epoch(prev_time,
timezone="kr")
end_time = Global.convert_local_datetime_to_epoch(cur_time,
timezone="kr")
# draw iyo_config for bal & factor_setting
iyo_config = Global.read_iyo_setting_config(coin_name)
# FIXME: 빗썸 등등 거래소 생긴 날부터는 밑에 주석 쓰기
# rfab_combi_list = Global.get_rfab_combination_list(coin_name)
rfab_combi_list = list(it.combinations(["okcoin", "coinnest"], 2))
for _combi in rfab_combi_list:
logging.critical(
"[%s-%s-%s] IYO conducting -> start_time: %s, end_time: %s" %
(coin_name.upper(), str(_combi[0]).upper(), str(
_combi[1]).upper(), start_time, end_time))
# set settings, bal_fact_settings, factor_settings
settings = TradeSettingConfig.get_settings(
mm1_name=_combi[0],
mm2_name=_combi[1],
target_currency=coin_name,
start_time=start_time,
end_time=end_time,
division=iyo_config["division"],
depth=iyo_config["depth"],
consecution_time=iyo_config["consecution_time"],
is_virtual_mm=True)
bal_factor_settings = TradeSettingConfig.get_bal_fact_settings(
iyo_config["krw_seq_end"], iyo_config["coin_seq_end"])
factor_settings = TradeSettingConfig.get_factor_settings(
_combi[0], _combi[1], coin_name,
iyo_config["max_trade_coin_end"], iyo_config["threshold_end"],
iyo_config["appx_unit_coin_price"])
iyo_result = IntegratedYieldOptimizer.run(settings,
bal_factor_settings,
factor_settings)
# finally save to mongoDB
if len(iyo_result) > 0:
db_client["statistics"]["iyo"].insert_many(iyo_result)
else:
logging.critical(
"There was no oppty!! Skipping to next combination!")
continue
logging.warning("Nohup done, now conducting next time set!!")
prev_time = cur_time
def add_missing_item_with_plain_copy_prev(a_db: str, a_col: str, b_db: str,
b_col: str, start_time: int,
end_time: int):
db_client = SharedMongoClient.instance()
a_target_col = db_client[a_db][a_col]
b_target_col = db_client[b_db][b_col]
redefined_start_time = start_time
while True:
retry_flag = False
a_cursor = a_target_col.find({
"requestTime": {
"$gte": redefined_start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])
b_cursor = b_target_col.find({
"requestTime": {
"$gte": redefined_start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])
a_count = a_cursor.count()
b_count = b_cursor.count()
logging.info("Cursor count: a %d, b %d" % (a_count, b_count))
last_a_item = None
last_b_item = None
for a_item, b_item in zip_longest(a_cursor, b_cursor):
a_rt = a_item["requestTime"]
try:
b_rt = b_item["requestTime"]
except TypeError:
print(b_item)
b_rt = 0
if a_rt != b_rt:
logging.info("Diff: a_rt %d, b_rt %d " % (a_rt, b_rt))
if last_a_item is None or last_b_item is None:
raise Exception(
"At least the first occurrence should be a valid pair!"
)
is_b_older = a_rt > b_rt
if is_b_older:
last_a_item["requestTime"] = b_rt
logging.info("Adding %d item in a_col..." % b_rt)
a_target_col.insert_one(last_a_item)
else:
# if a is older
last_b_item["requestTime"] = a_rt
logging.info("Adding %d item in b_col..." % a_rt)
b_target_col.insert_one(last_b_item)
# redefine start time for cursor re-request
redefined_start_time = min(a_rt, b_rt)
retry_flag = True
break
else:
last_a_item = dict(a_item)
del last_a_item["_id"]
last_b_item = dict(b_item)
del last_b_item["_id"]
if retry_flag:
a_cursor.close()
b_cursor.close()
continue
else:
break
def match_request_time_in_orderbook_entry(control_db: str, target_db: str,
col_name: str, start_time: int,
end_time: int):
db_client = SharedMongoClient.instance()
control_col = db_client[control_db][col_name]
target_col = db_client[target_db][col_name]
ctrl_data_set: Cursor = control_col.find({
"requestTime": {
"$gte": start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])
# get first nearest request time from control db
first_ctrl_rq = ctrl_data_set[0]["requestTime"]
# get first and second request time from target db
trgt_data_set = list(
target_col.find({
"requestTime": {
"$gte": start_time,
"$lte": end_time
}
}).sort([("requestTime", 1)])[:2])
# first target requestTime
first_trgt_rq = trgt_data_set[0]["requestTime"]
# second target requestTime
second_trgt_rq = trgt_data_set[1]["requestTime"]
# calc difference between control reqTime and target reqTimes
ctrl_first_trgt_first_diff = abs(first_ctrl_rq - first_trgt_rq)
ctrl_first_trgt_second_diff = abs(first_ctrl_rq - second_trgt_rq)
# if first in target is nearer to first in control, set first in target as starting point
if ctrl_first_trgt_first_diff < ctrl_first_trgt_second_diff:
trgt_start_rq = first_trgt_rq
# if second in target is nearer to first in control, set second in target as starting point
elif ctrl_first_trgt_first_diff > ctrl_first_trgt_second_diff:
trgt_start_rq = second_trgt_rq
else:
raise Exception(
"Difference is same, please Manually check the database and fix!!"
)
# get count of data from control db
ctrl_data_count = ctrl_data_set.count()
# get same count of data from target db with the starting point as start time and without end time
trgt_data_set: Cursor = target_col.find({
"requestTime": {
"$gte": trgt_start_rq
}
}).sort([("requestTime", 1)]).limit(ctrl_data_count)
trgt_data_count = trgt_data_set.count(with_limit_and_skip=True)
logging.info("ctrl count count: %d, trgt: %d" %
(ctrl_data_count, trgt_data_count))
assert (ctrl_data_count == trgt_data_count)
last_index = ctrl_data_count - 1
ctrl_last_rq = ctrl_data_set[last_index]["requestTime"]
trgt_last_rq = trgt_data_set[last_index]["requestTime"]
assert (abs(ctrl_last_rq - trgt_last_rq) < 3)
# loop through both
# update target's request time with control's request
for ctrl_data, trgt_data in zip(ctrl_data_set, trgt_data_set):
ctrl_rq = ctrl_data["requestTime"]
trgt_rq = trgt_data["requestTime"]
logging.info("ctrl_rqt: %d, trgt_rqt: %d" % (ctrl_rq, trgt_rq))
if trgt_rq == ctrl_rq:
continue
SharedMongoClient._async_update(target_col,
{"requestTime": trgt_rq},
{"$set": {
"requestTime": ctrl_rq
}})