def QA_SU_save_okex_min(frequency='60', ui_log=None, ui_progress=None):
"""
Save OKEx min kline 分钟线数据，统一转化字段保存数据为 crypto_asset_min
"""
market = 'okex'
symbol_list = QA_fetch_crypto_asset_list(market='okex')
col = DATABASE.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info('Starting DOWNLOAD PROGRESS of min Klines from OKEx... ',
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
# 上架仅处理交易对
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_info['symbol'], index, len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_info['symbol'],
'market': symbol_info['market'],
'type': OKEx2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_info['symbol'],
OKEx2QA_FREQUENCY_DICT[frequency],
market='okex')[::-1]
else:
start_time = OKEx_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
QA_util_datetime_to_Unix_timestamp(start_time),
QA_util_datetime_to_Unix_timestamp(end), '{} 到 {}'.format(
start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = missing_data_list[i][expected]
reqParams['to'] = missing_data_list[i][between]
if (reqParams['to'] >
(QA_util_datetime_to_Unix_timestamp() + 3600)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_info['symbol'], frequency,
QA_util_print_timestamp(reqParams['to']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_info['symbol'],
OKEx2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_okex_kline_min(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_func=QA_SU_save_data_okex_callback)
if data is None:
QA_util_log_info(
'SYMBOL "{}" from {} to {} has no MORE data'.format(
symbol_info['symbol'],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info('DOWNLOAD PROGRESS of min Klines from OKEx accomplished.',
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_huobi_min(
frequency=CandlestickInterval.MIN1,
fetch_range='all',
ui_log=None,
ui_progress=None,
):
"""
下载火币K线分钟数据，统一转化字段保存数据为 cryptocurrency_min
"""
symbol_template = huobi_SYMBOL
symbol_list = QA_fetch_cryptocurrency_list(huobi_EXCHANGE)
print("XXX", huobi_EXCHANGE, symbol_list)
col = DATABASE.cryptocurrency_min
col.create_index([("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of min Klines from {:s}... '.format(
huobi_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
if ((fetch_range != 'all')
and (symbol_info['symbol'] not in fetch_range)):
# Process save_range[] only
continue
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_template.format(symbol_info['symbol']), index,
len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_template.format(symbol_info['symbol']),
'type': Huobi2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
)[::-1]
else:
start_time = huobi_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
int(QA_util_datetime_to_Unix_timestamp(start_time)),
int(QA_util_datetime_to_Unix_timestamp(end)),
'{} to {}'.format(start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = int(missing_data_list[i][expected])
reqParams['to'] = int(missing_data_list[i][between])
if (reqParams['from'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
# 跳到下一个时间段
continue
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_huobi_kline_subscription(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_func=QA_SU_save_data_huobi_callback)
if data is None:
QA_util_log_info(
'SYMBOL "{}" from {} to {} has no MORE data'.format(
symbol_template.format(symbol_info['symbol']),
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info(
'DOWNLOAD PROGRESS of min Klines from {:s} accomplished.'.format(
huobi_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_bitfinex_day(frequency, ui_log=None, ui_progress=None):
"""
Save Bitfinex day kline 日线数据，统一转化字段保存数据为 crypto_asset_day
"""
symbol_template = Bitfinex_SYMBOL
symbol_list = QA_fetch_cryptocurrency_list(Bitfinex_EXCHANGE)
col = DATABASE.cryptocurrency_day
col.create_index([("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format(
Bitfinex_EXCHANGE),
ui_log=ui_log,
ui_progress=ui_progress)
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_template.format(symbol_info['symbol']), index,
len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_template.format(symbol_info['symbol']),
}
ref = col.find(query_id).sort('date_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['date_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
)[::-1]
else:
start_time = Bitfinex_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
miss_kline = pd.DataFrame(
[[
int(QA_util_datetime_to_Unix_timestamp(start_time)),
int(QA_util_datetime_to_Unix_timestamp(end)),
'{} to {}'.format(start_time, end)
]],
columns=['expected', 'between', 'missing'])
missing_data_list = miss_kline.values
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = int(missing_data_list[i][expected])
reqParams['to'] = int(missing_data_list[i][between])
if (reqParams['from'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
QA_util_log_info(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}'
.format(
symbol_template.format(symbol_info['symbol']),
frequency,
QA_util_print_timestamp(reqParams['from']),
QA_util_print_timestamp(
QA_util_datetime_to_Unix_timestamp())))
# 跳到下一个时间段
continue
QA_util_log_info(
'Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}'.format(
symbol_template.format(symbol_info['symbol']),
Bitfinex2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
data = QA_fetch_bitfinex_kline(
symbol_info['symbol'],
time.mktime(start_time.utctimetuple()),
time.mktime(end.utctimetuple()),
frequency,
callback_func=QA_SU_save_data_bitfinex_callback)
if data is None:
QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format(
symbol_template.format(symbol_info['symbol']),
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)),
ui_log=ui_log,
ui_progress=ui_progress)
continue
QA_util_log_info(
'DOWNLOAD PROGRESS of day Klines from Bitfinex accomplished.',
ui_log=ui_log,
ui_progress=ui_progress)
def QA_SU_save_huobi_min(client=DATABASE,
ui_log=None,
ui_progress=None,
frequency=CandlestickInterval.MIN1,
fetch_range='all'):
"""
下载火币K线分钟数据，统一转化字段保存数据为 crypto_asset_min
"""
def QA_SU_save_data_huobi_min_callback(data, symbol=''):
"""
异步获取数据回调用的 MongoDB 存储函数
"""
market = 'huobi'
QA_util_log_info(
'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[0]['time_stamp'])),
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[-1]['time_stamp'])),
len(data)),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol,
'market': market,
'type': Huobi2QA_FREQUENCY_DICT[frequency],
'time_stamp': {
'$in': list(map(lambda x: x['time_stamp'], data))
}
}
if (col.count_documents(query_id) > 0):
# 删掉重复数据
col.delete_many(query_id)
try:
col.insert_many(data)
except:
QA_util_log_expection(
'QA_SU_save_huobi_day():Insert_many(kline) to {} got Exception {}'
.format(symbol_info['symbol'], len(data)))
pass
symbol_list = QA_fetch_crypto_asset_list('huobi')
col = client.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
end = datetime.datetime.now(tzutc())
QA_util_log_info(
'Starting DOWNLOAD PROGRESS of min Klines from huobi.pro... ')
for index in range(len(symbol_list)):
symbol_info = symbol_list.iloc[index]
if ((fetch_range != 'all')
and (symbol_info['symbol'] not in fetch_range)):
# Process save_range[] only
continue
QA_util_log_info('The "{}" #{} of total in {}'.format(
symbol_info['symbol'], index, len(symbol_list)),
ui_log=ui_log,
ui_progress=ui_progress)
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(index / len(symbol_list) * 100))[0:4] + '%'),
ui_log=ui_log,
ui_progress=ui_progress)
query_id = {
"symbol": symbol_info['symbol'],
'market': symbol_info['market'],
'type': Huobi2QA_FREQUENCY_DICT[frequency]
}
ref = col.find(query_id).sort('time_stamp', -1)
if (col.count_documents(query_id) > 0):
start_stamp = ref.next()['time_stamp']
start_time = datetime.datetime.fromtimestamp(start_stamp + 1,
tz=tzutc())
QA_util_log_info(
'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
# 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
missing_data_list = QA_util_find_missing_kline(
symbol_info['symbol'],
Huobi2QA_FREQUENCY_DICT[frequency])[::-1]
if len(missing_data_list) > 0:
# 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
expected = 0
between = 1
missing = 2
reqParams = {}
for i in range(len(missing_data_list)):
reqParams['from'] = missing_data_list[i][expected]
reqParams['to'] = missing_data_list[i][between]
if (reqParams['to'] >
(QA_util_datetime_to_Unix_timestamp() + 120)):
# 出现“未来”时间，一般是默认时区设置错误造成的
raise Exception(
'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s}({:d}) but current time is {}'
.format(symbol_info['symbol'], frequency,
missing_data_list[i][missing],
missing_data_list[i][between],
QA_util_datetime_to_Unix_timestamp()))
QA_util_log_info(
'Fetch %s missing %s kline：%s 到 %s' %
(symbol_info['symbol'], frequency,
QA_util_timestamp_to_str(
missing_data_list[i][expected])[2:16],
QA_util_timestamp_to_str(
missing_data_list[i][between])[2:16]))
QA_fetch_huobi_kline_subscription(
symbol_info['symbol'],
start_time=reqParams['from'],
end_time=reqParams['to'],
frequency=frequency,
callback_save_data_func=QA_SU_save_data_huobi_callback)
else:
start_time = huobi_MIN_DATE
QA_util_log_info(
'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
data = QA_fetch_huobi_kline(
symbol_info['symbol'],
time.mktime(start_time.utctimetuple()),
time.mktime(end.utctimetuple()),
frequency=frequency,
callback_save_data_func=QA_SU_save_data_huobi_min_callback)
if data is None:
QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
QA_util_timestamp_to_str(start_time),
QA_util_timestamp_to_str(end)))
continue
QA_util_log_info(
'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format(
symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency],
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[0]['time_stamp'])),
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data[-1]['time_stamp'])),
len(data)))
QA_util_log_info(
'DOWNLOAD PROGRESS of min Klines from huobi.pro accomplished.')
