def QA_fetch_huobi_kline_subscription(symbol, start_time, end_time, frequency,
callback_save_data_func):
"""
Get the symbol‘s candlestick data by subscription
"""
def print_timestamp(ts_epoch):
return '{}({})'.format(
QA_util_timestamp_to_str(ts_epoch)[2:16], ts_epoch)
reqParams = {}
reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency]
reqParams['to'] = end_time
if (reqParams['to'] > QA_util_datetime_to_Unix_timestamp()):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set'
)
data = []
requested_counter = 1
sub_client = QA_Fetch_Huobi(
callback_save_data_func=callback_save_data_func,
find_missing_kline_func=QA_util_find_missing_kline)
while (reqParams['to'] > start_time):
if (reqParams['to'] > QA_util_datetime_to_Unix_timestamp()):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set'
)
retries = 1
while (retries != 0):
try:
frame = sub_client.run_request_historical_kline(
symbol, frequency, reqParams['from'], reqParams['to'],
requested_counter)
if (frame is None):
# 火币网的 WebSocket 接口机制很奇特，返回len(data)==0
# 就说明已经超越这个交易对的上架时间，不再有更多数据了。
# 当前 Symbol Klines 抓取已经结束了
return None
else:
retries = 0
except Exception:
retries = retries + 1
if (retries % 6 == 0):
print(ILOVECHINA)
print("Retry request_historical_kline #{}".format(retries - 1))
time.sleep(0.5)
if (retries == 0):
# 等待3秒，请求下一个时间段的批量K线数据
reqParams['to'] = reqParams['from'] - 1
reqParams[
'from'] = reqParams['from'] - FREQUENCY_SHIFTING[frequency]
requested_counter = requested_counter + 1
# 这一步冗余，如果是开启实时抓取会自动被 WebSocket.on_messgae_callback 事件处理函数保存，
# 但不确定不开启实时行情抓取会不会绑定on_messgae事件，所以保留冗余。
callback_save_data_func(data=frame, freq=frequency)
time.sleep(0.5)
return data
def QA_fetch_bitfinex_kline(symbol,
start_time,
end_time,
frequency,
callback_func=None):
"""
Get the latest symbol‘s candlestick data
时间倒序切片获取算法，是各大交易所获取1min数据的神器，因为大部分交易所直接请求跨月跨年的1min分钟数据
会直接返回空值，只有将 start_epoch，end_epoch 切片细分到 200/300 bar 以内，才能正确返回 kline，
火币和okex，OKEx 均为如此，直接用跨年时间去直接请求上万bar 的 kline 数据永远只返回最近200条数据。
"""
market = 'okex'
datas = list()
reqParams = {}
reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency]
reqParams['to'] = end_time
while (reqParams['to'] > start_time):
if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \
((reqParams['from'] > reqParams['to'])):
# 出现“未来”时间，一般是默认时区设置，或者时间窗口滚动前移错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol, frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
klines = QA_fetch_okex_kline_with_auto_retry(
symbol,
reqParams['from'],
reqParams['to'],
frequency,
)
if (len(klines) == 0) or \
('error' in klines):
# 出错放弃
break
reqParams['to'] = reqParams['from'] - 1
reqParams['from'] = reqParams['from'] - FREQUENCY_SHIFTING[frequency]
if (klines is None) or \
((len(datas) > 0) and (klines[-1][0] == datas[-1][0])):
# 没有更多数据
break
datas.extend(klines)
if (callback_func is not None):
frame = format_okex_data_fields(klines, symbol, frequency)
callback_func(frame, OKEx2QA_FREQUENCY_DICT[frequency])
if len(datas) == 0:
return None
# 归一化数据字段，转换填充必须字段，删除多余字段
frame = format_okex_data_fields(datas, symbol, frequency)
return frame
def QA_fetch_bitmex_kline_min(
symbol,
start_time,
end_time,
frequency,
callback_func=None
):
"""
Get the latest symbol‘s candlestick data with time slices
时间倒序切片获取算法，是各大交易所获取1min数据的神器，因为大部分交易所直接请求跨月跨年的1min分钟数据
会直接返回空值，只有将 start_epoch，end_epoch 切片细分到 1000 bar 以内，才能正确返回 kline，
火币和Bitmex，OKEx 均为如此，用上面那个函数的方式去直接请求上万bar 的分钟 kline 数据是不会有结果的。
"""
reqParams = {}
reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency]
reqParams['to'] = end_time
requested_counter = 1
retries = 1
datas = list()
while (reqParams['to'] > start_time):
if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \
((reqParams['from'] > reqParams['to'])):
# 出现“未来”时间，一般是默认时区设置，或者时间窗口滚动前移错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol,
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp()
)
)
)
# 跳到下一个时间段
reqParams['to'] = int(reqParams['from'] - 1)
reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency])
continue
klines = QA_fetch_bitmex_kline_with_auto_retry(
symbol,
reqParams['from'],
reqParams['to'],
frequency,
)
if (len(klines) == 0) or \
('error' in klines):
# 出错放弃
break
reqParams['to'] = int(reqParams['from'] - 1)
reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency])
if (callback_func is not None):
frame = format_btimex_data_fields(klines, frequency)
callback_func(frame, Bitmex2QA_FREQUENCY_DICT[frequency])
if (len(klines) == 0):
return None
def rolling_messenger(stock_params):
"""
滚动查询和发送系统消息到关联的微信号上面
"""
alive_conter = 0
while (True):
stock_params['btcusdt']['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
#stock_params['btcusdt']['from'] =
#QA_util_timestamp_to_str(datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params['btcusdt'])
time.sleep(1)
stock_params['htusdt']['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params['htusdt'])
time.sleep(1)
stock_params['ethusdt']['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params['ethusdt'])
time.sleep(1)
stock_params["eosusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["eosusdt"])
time.sleep(1)
stock_params["etcusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["etcusdt"])
time.sleep(1)
stock_params["bchusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["bchusdt"])
time.sleep(1)
stock_params["dashusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["dashusdt"])
time.sleep(1)
stock_params["ltcusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["ltcusdt"])
time.sleep(1)
stock_params["xmrusdt"]['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
order_messenger(stock_params["xmrusdt"])
stock_params['btcusdt']['to'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))
stock_params['btcusdt']['from'] = QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))) - timedelta(minutes=5))
stock_params['btcusdt'] = check_fetch_huobi(stock_params['btcusdt'])
time.sleep(1)
if ((QA_util_datetime_to_Unix_timestamp(
datetime.now(timezone(timedelta(hours=8)))) - alive_conter) >=
60):
alive_conter = QA_util_datetime_to_Unix_timestamp(
datetime.now(timezone(timedelta(hours=8))))
print(u'%s WeChat/微信/Server酱信息推送提醒 Alive...' %
(QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8))))))
def check_fetch_huobi(stock_conf, time_window=2):
"""
检查实时行情数据抓取是否中断，如果交易中断则微信发送提醒
"""
stock_conf['from'] = (datetime.strptime(stock_conf['to'] + ' +0800',
'%Y-%m-%d %H:%M:%S %z') -
timedelta(hours=time_window)).astimezone(
timezone(timedelta(
hours=8))).strftime('%Y-%m-%d %H:%M:%S')
data1min = QA_fetch_crypto_min_adv(
[stock_conf['stock_code']],
stock_conf['from'],
stock_conf['to'],
frequence=FREQUENCE.ONE_MIN,
)
Period_Time = 60
if (len(data1min.data) != 0):
utc_timeline = int(data1min.data.iloc[-1].timestamp)
time1 = datetime(1970, 1, 1,
tzinfo=timezone.utc) + timedelta(seconds=utc_timeline)
else:
time1 = datetime(1970, 1, 1, tzinfo=timezone.utc)
time2 = datetime.now(timezone(timedelta(hours=8)))
if (len(data1min.data) == 0) or \
(Period_Time * 3 < (time2 - time1).total_seconds()):
# * 代表非实时数据，已经超出当前时间区间
if ((time2 - time1).total_seconds() > 14400):
msg = u"实时行情 MariaDB 数据库已经走丢了，已经超过 %d 小时没有更新，请主人检查 DS1517+ 情况。—— %s" % (
time_window,
QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8)))),
)
else:
msg = u"huobi.pro 实时行情数据已经超过 %d 秒没有更新，请主人检查那台宇宙超级无敌的主机和海外网络连接情况。—— %s" % (
(time2 - time1).total_seconds(),
QA_util_timestamp_to_str(
datetime.now(timezone(timedelta(hours=8)))),
)
if ((QA_util_datetime_to_Unix_timestamp(
datetime.now(timezone(timedelta(hours=8)))) -
stock_conf['alert_counter']) >= 300):
stock_conf['alert_counter'] = QA_util_datetime_to_Unix_timestamp(
datetime.now(timezone(timedelta(hours=8))))
send_servercha(msg)
#send_wechat(msg)
print(msg)
return stock_conf
def QA_util_find_missing_kline(symbol,
freq,
start_epoch=datetime(2017,
10,
1,
tzinfo=tzutc()),
tzlocalize='Asia/Shanghai'):
"""
查找24小时不间断的连续交易市场中缺失的 kline 历史数据，生成缺失历史数据时间段
"""
FREQUENCE_PERIOD_TIME = {
FREQUENCE.ONE_MIN: 60,
FREQUENCE.FIVE_MIN: 300,
FREQUENCE.FIFTEEN_MIN: 900,
FREQUENCE.THIRTY_MIN: 1800,
FREQUENCE.SIXTY_MIN: 3600,
FREQUENCE.HOUR: 3600,
FREQUENCE.DAY: 86400,
}
if (freq != FREQUENCE.DAY):
col = DATABASE.cryptocurrency_min
col.create_index([("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
col.create_index([("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)])
col.create_index([('time_stamp', pymongo.ASCENDING)])
# 查询历史数据
query_id = {"symbol": symbol, 'type': freq}
refcount = col.count_documents(query_id)
_data = []
cursor = col.find(query_id).sort('time_stamp', 1)
for item in cursor:
_data.append([
str(item['symbol']), item['time_stamp'], item['date'],
item['datetime'], item['type']
])
_data = pd.DataFrame(
_data,
columns=['symbol', 'time_stamp', 'date', 'datetime', 'type'])
_data = _data.set_index(pd.DatetimeIndex(_data['datetime']),
drop=False)
else:
col = DATABASE.cryptocurrency_day
col.create_index([("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
# 查询是否新 tick
query_id = {"symbol": symbol}
refcount = col.count_documents(query_id)
cursor = col.find(query_id).sort('time_stamp', 1)
_data = []
for item in cursor:
_data.append([
str(item['symbol']), item['time_stamp'], item['date'],
item['datetime']
])
_data = pd.DataFrame(
_data, columns=['symbol', 'time_stamp', 'date',
'datetime']).drop_duplicates()
_data['date'] = pd.to_datetime(_data['date'], utc=False)
_data = _data.set_index(pd.DatetimeIndex(_data['date']), drop=False)
if (freq != FREQUENCE.DAY):
# cryptocurrency_min 中的 Date/Datetime 字段均为北京时间
leak_datetime = pd.date_range(_data.index.min(),
_data.index.max(),
freq=freq).difference(
_data.index).tz_localize(tzlocalize)
if (int(_data.iloc[0].time_stamp) >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol, freq,
QA_util_print_timestamp(_data.iloc[0].time_stamp),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
else:
leak_datetime = pd.date_range(_data.index.min(),
_data.index.max(),
freq='1D').difference(
_data.index).tz_localize(tzlocalize)
if (int(time.mktime(start_epoch.utctimetuple())) > int(
_data.iloc[0].time_stamp)):
miss_kline = pd.DataFrame(columns=['expected', 'between', 'missing'])
else:
miss_kline = pd.DataFrame([[
int(time.mktime(start_epoch.utctimetuple())),
int(_data.iloc[0].time_stamp), '{} to {}'.format(
start_epoch, _data.iloc[0].date)
]],
columns=['expected', 'between', 'missing'])
expected = None
for x in range(0, len(leak_datetime)):
if (expected is None):
expected = int(leak_datetime[x].timestamp())
if ((expected is not None) and (x > 1) and (int(leak_datetime[x].timestamp()) != int(leak_datetime[x - 1].timestamp() + FREQUENCE_PERIOD_TIME[freq]))) or \
((expected is not None) and (x > 1) and (x == len(leak_datetime) - 1)):
between = int(leak_datetime[x - 1].timestamp() +
FREQUENCE_PERIOD_TIME[freq])
miss_kline = miss_kline.append(
{
'expected':
int(expected),
'between':
int(between),
'missing':
'{} to {}'.format(
pd.to_datetime(expected,
unit='s').tz_localize('Asia/Shanghai'),
pd.to_datetime(between,
unit='s').tz_localize('Asia/Shanghai'))
},
ignore_index=True)
expected = int(leak_datetime[x].timestamp())
if (int(_data.iloc[-1].time_stamp) + 1 < int(
QA_util_datetime_to_Unix_timestamp())):
miss_kline = miss_kline.append(
{
'expected':
int(_data.iloc[-1].time_stamp) + 1,
'between':
int(QA_util_datetime_to_Unix_timestamp()),
'missing':
'{} to {}'.format(
int(_data.iloc[0].time_stamp) + 1,
QA_util_datetime_to_Unix_timestamp())
},
ignore_index=True)
miss_kline.sort_values(by='expected', ascending=True, inplace=True)
if (len(miss_kline) > 0):
if (miss_kline.iloc[0].expected > QA_util_datetime_to_Unix_timestamp()) and \
(miss_kline.iloc[0].between > QA_util_datetime_to_Unix_timestamp()):
miss_kline.drop(miss_kline.iloc[0], inplace=True)
return miss_kline.values
def QA_SU_save_okex_min(frequency='60', ui_log=None, ui_progress=None):
"""
Save OKEx min kline 分钟线数据，统一转化字段保存数据为 crypto_asset_min
"""
market = 'okex'
symbol_list = QA_fetch_crypto_asset_list(market='okex')
col = DATABASE.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info('Starting DOWNLOAD PROGRESS of min Klines from OKEx... ',
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
# 上架仅处理交易对
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_info['symbol'], index, len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_info['symbol'],
'market': symbol_info['market'],
'type': OKEx2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_info['symbol'],
OKEx2QA_FREQUENCY_DICT[frequency],
market='okex')[::-1]
else:
start_time = OKEx_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
QA_util_datetime_to_Unix_timestamp(start_time),
QA_util_datetime_to_Unix_timestamp(end), '{} 到 {}'.format(
start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = missing_data_list[i][expected]
reqParams['to'] = missing_data_list[i][between]
if (reqParams['to'] >
(QA_util_datetime_to_Unix_timestamp() + 3600)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_info['symbol'], frequency,
QA_util_print_timestamp(reqParams['to']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_info['symbol'],
OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_okex_kline_min(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_func=QA_SU_save_data_okex_callback)
if data is None:
QA_util_log_info(
'SYMBOL "{}" from {} to {} has no MORE data'.format(
symbol_info['symbol'],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info('DOWNLOAD PROGRESS of min Klines from OKEx accomplished.',
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_bitfinex_day(frequency, ui_log=None, ui_progress=None):
"""
Save Bitfinex day kline 日线数据，统一转化字段保存数据为 crypto_asset_day
"""
symbol_template = Bitfinex_SYMBOL
symbol_list = QA_fetch_cryptocurrency_list(Bitfinex_EXCHANGE)
col = DATABASE.cryptocurrency_day
col.create_index([("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format(
Bitfinex_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_template.format(symbol_info['symbol']), index,
len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_template.format(symbol_info['symbol']),
}
ref = col.find(query_id).sort('date_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['date_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
)[::-1]
else:
start_time = Bitfinex_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
int(QA_util_datetime_to_Unix_timestamp(start_time)),
int(QA_util_datetime_to_Unix_timestamp(end)),
'{} to {}'.format(start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = int(missing_data_list[i][expected])
reqParams['to'] = int(missing_data_list[i][between])
if (reqParams['from'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
# 跳到下一个时间段
continue
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_bitfinex_kline(
symbol_info['symbol'],
time.mktime(start_time.utctimetuple()),
time.mktime(end.utctimetuple()),
frequency,
callback_func=QA_SU_save_data_bitfinex_callback)
if data is None:
QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format(
symbol_template.format(symbol_info['symbol']),
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
continue
QA_util_log_info(
'DOWNLOAD PROGRESS of day Klines from Bitfinex accomplished.',
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_bitmex_day(frequency='1d', ui_log=None, ui_progress=None):
"""
获取 bitmex K线 日线数据，统一转化字段保存数据为 crypto_asset_day
"""
symbol_template = bitmex_SYMBOL
symbol_list = QA_fetch_cryptocurrency_list(bitmex_EXCHANGE)
col = DATABASE.cryptocurrency_day
col.create_index([("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format(
bitmex_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_template.format(symbol_info['symbol']), index,
len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_template.format(symbol_info['symbol']),
}
ref = col.find(query_id).sort('date_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['date_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitmex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
else:
start_time = symbol_info.get('listing', "2018-01-01T00:00:00Z")
start_time = parse(start_time)
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitmex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
data = QA_fetch_bitmex_kline(
symbol_info['symbol'],
QA_util_datetime_to_Unix_timestamp(start_time),
QA_util_datetime_to_Unix_timestamp(end),
frequency,
callback_func=QA_SU_save_data_bitmex_callback)
if data is None:
QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format(
symbol_template.format(symbol_info['symbol']),
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
continue
QA_util_log_info(
'DOWNLOAD PROGRESS of day Klines from {:s} accomplished.'.format(
bitmex_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
def run(initalParams):
"""
运行于多线程的ws 请求发送，每3秒请求一次K线图数据，一次请求240条（的时间段），从参数设定的开始时间，请求到终止时间段为止
"""
# missing data key indexes
expected = 0
between = 1
missing = 2
requested_counter = 1
reqParams = {
'req': initalParams['req'],
}
# 采用时间倒序抓取，huobi WebSocket api 非常有个性，基本上遵循2个规则：
# 'from' 'to' 范围内没交易数据（交易对未上架）——返回 0 数据，
# 'from' 'to' 范围内超过300个bar，——返回 0 数据，一个交易对最早是什么时候上架的——无法查询到
# 所以选择倒序算法，从最近的时间开始补历史数据，'from' 'to' 请求范围一直向前递减到请求不到数据为止。
missing_data_list = initalParams['missing'][::-1]
for i in range(len(missing_data_list)):
reqParams['from'] = int(missing_data_list[i][between] -
initalParams['shifting_time'])
reqParams['to'] = int(missing_data_list[i][between])
if (reqParams['to'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
initalParams['req'],
QA_util_print_timestamp(reqParams['to']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
QA_util_log_info('Fetch %s missing kline：%s 到 %s' %
(initalParams['req'],
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
while (reqParams['to'] > missing_data_list[i][expected]):
if (self.__batchReqJobs[initalParams['req']].Status ==
QA_Fetch_Job_Status.FINISHED):
# 抓取已经结束了
return True
if (self.__batchReqJobs[initalParams['req']].Status ==
QA_Fetch_Job_Status.READY):
reqParams['id'] = "%s_#%d" % (initalParams['id'],
requested_counter)
if (reqParams['from'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s} at {:s} but current time is {}'
.format(
initalParams['req'],
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
self.__batchReqJobs[
initalParams['req']].Request.append(reqParams)
self.send_message(
reqParams,
'request kline {:s} part#{:d} {:s} to {:s}'.format(
initalParams['req'], requested_counter,
QA_util_timestamp_to_str(
reqParams['from'])[2:16],
QA_util_timestamp_to_str(
reqParams['to'])[2:16]))
# 等待3秒，请求下一个时间段的批量K线数据
reqParams['to'] = int(reqParams['from'] - 1)
reqParams['from'] = int(reqParams['from'] -
initalParams['shifting_time'])
requested_counter = requested_counter + 1
# 锁定线程，等待回复，避免快速频繁重复请求，会被ban IP的
self.__batchReqJobs[initalParams['req']].setStatus(
QA_Fetch_Job_Status.RUNNING)
else:
# WebSocket请求发出后等待没有回复无需特别处理，一般是SSR/SSL断线，会自动重连，继续补缺失数据
time.sleep(10)
self.send_message(
reqParams,
'request kline {:s} part#{:d} {:s} to {:s}'.format(
initalParams['req'], requested_counter,
QA_util_timestamp_to_str(
reqParams['from'])[2:16],
QA_util_timestamp_to_str(
reqParams['to'])[2:16]))
pass
time.sleep(3)
def QA_SU_save_huobi_min(
frequency=CandlestickInterval.MIN1,
fetch_range='all',
ui_log=None,
ui_progress=None,
):
"""
下载火币K线分钟数据，统一转化字段保存数据为 cryptocurrency_min
"""
symbol_template = huobi_SYMBOL
symbol_list = QA_fetch_cryptocurrency_list(huobi_EXCHANGE)
print("XXX", huobi_EXCHANGE, symbol_list)
col = DATABASE.cryptocurrency_min
col.create_index([("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of min Klines from {:s}... '.format(
huobi_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
if ((fetch_range != 'all')
and (symbol_info['symbol'] not in fetch_range)):
# Process save_range[] only
continue
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_template.format(symbol_info['symbol']), index,
len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_template.format(symbol_info['symbol']),
'type': Huobi2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
)[::-1]
else:
start_time = huobi_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
int(QA_util_datetime_to_Unix_timestamp(start_time)),
int(QA_util_datetime_to_Unix_timestamp(end)),
'{} to {}'.format(start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = int(missing_data_list[i][expected])
reqParams['to'] = int(missing_data_list[i][between])
if (reqParams['from'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
# 跳到下一个时间段
continue
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_huobi_kline_subscription(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_func=QA_SU_save_data_huobi_callback)
if data is None:
QA_util_log_info(
'SYMBOL "{}" from {} to {} has no MORE data'.format(
symbol_template.format(symbol_info['symbol']),
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info(
'DOWNLOAD PROGRESS of min Klines from {:s} accomplished.'.format(
huobi_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_huobi_min(client=DATABASE,
ui_log=None,
ui_progress=None,
frequency=CandlestickInterval.MIN1,
fetch_range='all'):
"""
下载火币K线分钟数据，统一转化字段保存数据为 crypto_asset_min
"""
def QA_SU_save_data_huobi_min_callback(data, symbol=''):
"""
异步获取数据回调用的 MongoDB 存储函数
"""
market = 'huobi'
QA_util_log_info(
'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[0]['time_stamp'])),
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[-1]['time_stamp'])),
len(data)),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol,
'market': market,
'type': Huobi2QA_FREQUENCY_DICT[frequency],
'time_stamp': {
'$in': list(map(lambda x: x['time_stamp'], data))
}
}
if (col.count_documents(query_id) > 0):
# 删掉重复数据
col.delete_many(query_id)
try:
col.insert_many(data)
except:
QA_util_log_expection(
'QA_SU_save_huobi_day():Insert_many(kline) to {} got Exception {}'
.format(symbol_info['symbol'], len(data)))
pass
symbol_list = QA_fetch_crypto_asset_list('huobi')
col = client.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of min Klines from huobi.pro... ')
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
if ((fetch_range != 'all')
and (symbol_info['symbol'] not in fetch_range)):
# Process save_range[] only
continue
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_info['symbol'], index, len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_info['symbol'],
'market': symbol_info['market'],
'type': Huobi2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_info['symbol'],
Huobi2QA_FREQUENCY_DICT[frequency])[::-1]
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = missing_data_list[i][expected]
reqParams['to'] = missing_data_list[i][between]
if (reqParams['to'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s}({:d}) but current time is {}'
.format(symbol_info['symbol'], frequency,
missing_data_list[i][missing],
missing_data_list[i][between],
QA_util_datetime_to_Unix_timestamp()))
QA_util_log_info(
'Fetch %s missing %s kline：%s 到 %s' %
(symbol_info['symbol'], frequency,
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
QA_fetch_huobi_kline_subscription(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_save_data_func=QA_SU_save_data_huobi_callback)
else:
start_time = huobi_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
data = QA_fetch_huobi_kline(
symbol_info['symbol'],
time.mktime(start_time.utctimetuple()),
time.mktime(end.utctimetuple()),
frequency=frequency,
callback_save_data_func=QA_SU_save_data_huobi_min_callback)
if data is None:
QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info(
'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[0]['time_stamp'])),
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[-1]['time_stamp'])),
len(data)))
QA_util_log_info(
'DOWNLOAD PROGRESS of min Klines from huobi.pro accomplished.')
def QA_fetch_huobi_kline_subscription(
symbol,
start_time,
end_time,
frequency,
callback_func
):
"""
Get the symbol‘s candlestick data by subscription
"""
reqParams = {}
reqParams['from'] = int(end_time - FREQUENCY_SHIFTING[frequency])
reqParams['to'] = int(end_time)
requested_counter = 1
sub_client = QA_Fetch_Huobi(
callback_save_data_func=callback_func,
find_missing_kline_func=QA_util_find_missing_kline
)
datas = list()
retries = 1
while (reqParams['to'] > start_time):
if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \
((reqParams['from'] > reqParams['to'])):
# 出现“未来”时间，一般是默认时区设置，或者时间窗口滚动前移错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol,
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp()
)
)
)
# 跳到下一个时间段
reqParams['to'] = int(reqParams['from'] - 1)
reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency])
continue
retries = 1
frame = None
while (retries != 0):
try:
frame = sub_client.run_request_historical_kline(
symbol,
frequency,
reqParams['from'],
reqParams['to'],
requested_counter
)
if (frame is None):
# 火币网的 WebSocket 接口机制很奇特，返回len(data)==0
# 就说明已经超越这个交易对的上架时间，不再有更多数据了。
# 当前 Symbol Klines 抓取已经结束了
return None
else:
retries = 0
except Exception:
retries = retries + 1
if (retries % 6 == 0):
print(ILOVECHINA)
print("Retry request_historical_kline #{}".format(retries - 1))
time.sleep(0.5)
if (retries == 0):
# 等待3秒，请求下一个时间段的批量K线数据
reqParams['to'] = int(reqParams['from'] - 1)
reqParams['from'
] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency])
requested_counter = requested_counter + 1
# 这一步冗余，如果是开启实时抓取会自动被 WebSocket.on_messgae_callback 事件处理函数保存，
# 但不确定不开启实时行情抓取会不会绑定on_messgae事件，所以保留冗余。
callback_func(data=frame, freq=frequency)
time.sleep(0.5)
if ((retries == 0) and (len(frame) > 0)):
return frame
else:
return None