def QA_fetch_huobi_kline_subscription(symbol, start_time, end_time, frequency, callback_save_data_func): """ Get the symbol‘s candlestick data by subscription """ def print_timestamp(ts_epoch): return '{}({})'.format( QA_util_timestamp_to_str(ts_epoch)[2:16], ts_epoch) reqParams = {} reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency] reqParams['to'] = end_time if (reqParams['to'] > QA_util_datetime_to_Unix_timestamp()): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set' ) data = [] requested_counter = 1 sub_client = QA_Fetch_Huobi( callback_save_data_func=callback_save_data_func, find_missing_kline_func=QA_util_find_missing_kline) while (reqParams['to'] > start_time): if (reqParams['to'] > QA_util_datetime_to_Unix_timestamp()): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set' ) retries = 1 while (retries != 0): try: frame = sub_client.run_request_historical_kline( symbol, frequency, reqParams['from'], reqParams['to'], requested_counter) if (frame is None): # 火币网的 WebSocket 接口机制很奇特,返回len(data)==0 # 就说明已经超越这个交易对的上架时间,不再有更多数据了。 # 当前 Symbol Klines 抓取已经结束了 return None else: retries = 0 except Exception: retries = retries + 1 if (retries % 6 == 0): print(ILOVECHINA) print("Retry request_historical_kline #{}".format(retries - 1)) time.sleep(0.5) if (retries == 0): # 等待3秒,请求下一个时间段的批量K线数据 reqParams['to'] = reqParams['from'] - 1 reqParams[ 'from'] = reqParams['from'] - FREQUENCY_SHIFTING[frequency] requested_counter = requested_counter + 1 # 这一步冗余,如果是开启实时抓取会自动被 WebSocket.on_messgae_callback 事件处理函数保存, # 但不确定不开启实时行情抓取会不会绑定on_messgae事件,所以保留冗余。 callback_save_data_func(data=frame, freq=frequency) time.sleep(0.5) return data
def QA_fetch_bitfinex_kline(symbol, start_time, end_time, frequency, callback_func=None): """ Get the latest symbol‘s candlestick data 时间倒序切片获取算法,是各大交易所获取1min数据的神器,因为大部分交易所直接请求跨月跨年的1min分钟数据 会直接返回空值,只有将 start_epoch,end_epoch 切片细分到 200/300 bar 以内,才能正确返回 kline, 火币和okex,OKEx 均为如此,直接用跨年时间去直接请求上万bar 的 kline 数据永远只返回最近200条数据。 """ market = 'okex' datas = list() reqParams = {} reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency] reqParams['to'] = end_time while (reqParams['to'] > start_time): if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \ ((reqParams['from'] > reqParams['to'])): # 出现“未来”时间,一般是默认时区设置,或者时间窗口滚动前移错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol, frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) klines = QA_fetch_okex_kline_with_auto_retry( symbol, reqParams['from'], reqParams['to'], frequency, ) if (len(klines) == 0) or \ ('error' in klines): # 出错放弃 break reqParams['to'] = reqParams['from'] - 1 reqParams['from'] = reqParams['from'] - FREQUENCY_SHIFTING[frequency] if (klines is None) or \ ((len(datas) > 0) and (klines[-1][0] == datas[-1][0])): # 没有更多数据 break datas.extend(klines) if (callback_func is not None): frame = format_okex_data_fields(klines, symbol, frequency) callback_func(frame, OKEx2QA_FREQUENCY_DICT[frequency]) if len(datas) == 0: return None # 归一化数据字段,转换填充必须字段,删除多余字段 frame = format_okex_data_fields(datas, symbol, frequency) return frame
def QA_fetch_bitmex_kline_min( symbol, start_time, end_time, frequency, callback_func=None ): """ Get the latest symbol‘s candlestick data with time slices 时间倒序切片获取算法,是各大交易所获取1min数据的神器,因为大部分交易所直接请求跨月跨年的1min分钟数据 会直接返回空值,只有将 start_epoch,end_epoch 切片细分到 1000 bar 以内,才能正确返回 kline, 火币和Bitmex,OKEx 均为如此,用上面那个函数的方式去直接请求上万bar 的分钟 kline 数据是不会有结果的。 """ reqParams = {} reqParams['from'] = end_time - FREQUENCY_SHIFTING[frequency] reqParams['to'] = end_time requested_counter = 1 retries = 1 datas = list() while (reqParams['to'] > start_time): if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \ ((reqParams['from'] > reqParams['to'])): # 出现“未来”时间,一般是默认时区设置,或者时间窗口滚动前移错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol, frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp() ) ) ) # 跳到下一个时间段 reqParams['to'] = int(reqParams['from'] - 1) reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency]) continue klines = QA_fetch_bitmex_kline_with_auto_retry( symbol, reqParams['from'], reqParams['to'], frequency, ) if (len(klines) == 0) or \ ('error' in klines): # 出错放弃 break reqParams['to'] = int(reqParams['from'] - 1) reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency]) if (callback_func is not None): frame = format_btimex_data_fields(klines, frequency) callback_func(frame, Bitmex2QA_FREQUENCY_DICT[frequency]) if (len(klines) == 0): return None
def rolling_messenger(stock_params): """ 滚动查询和发送系统消息到关联的微信号上面 """ alive_conter = 0 while (True): stock_params['btcusdt']['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) #stock_params['btcusdt']['from'] = #QA_util_timestamp_to_str(datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params['btcusdt']) time.sleep(1) stock_params['htusdt']['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params['htusdt']) time.sleep(1) stock_params['ethusdt']['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params['ethusdt']) time.sleep(1) stock_params["eosusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["eosusdt"]) time.sleep(1) stock_params["etcusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["etcusdt"]) time.sleep(1) stock_params["bchusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["bchusdt"]) time.sleep(1) stock_params["dashusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["dashusdt"]) time.sleep(1) stock_params["ltcusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["ltcusdt"]) time.sleep(1) stock_params["xmrusdt"]['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) order_messenger(stock_params["xmrusdt"]) stock_params['btcusdt']['to'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))) stock_params['btcusdt']['from'] = QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8))) - timedelta(minutes=5)) stock_params['btcusdt'] = check_fetch_huobi(stock_params['btcusdt']) time.sleep(1) if ((QA_util_datetime_to_Unix_timestamp( datetime.now(timezone(timedelta(hours=8)))) - alive_conter) >= 60): alive_conter = QA_util_datetime_to_Unix_timestamp( datetime.now(timezone(timedelta(hours=8)))) print(u'%s WeChat/微信/Server酱信息推送提醒 Alive...' % (QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8))))))
def check_fetch_huobi(stock_conf, time_window=2): """ 检查实时行情数据抓取是否中断,如果交易中断则微信发送提醒 """ stock_conf['from'] = (datetime.strptime(stock_conf['to'] + ' +0800', '%Y-%m-%d %H:%M:%S %z') - timedelta(hours=time_window)).astimezone( timezone(timedelta( hours=8))).strftime('%Y-%m-%d %H:%M:%S') data1min = QA_fetch_crypto_min_adv( [stock_conf['stock_code']], stock_conf['from'], stock_conf['to'], frequence=FREQUENCE.ONE_MIN, ) Period_Time = 60 if (len(data1min.data) != 0): utc_timeline = int(data1min.data.iloc[-1].timestamp) time1 = datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=utc_timeline) else: time1 = datetime(1970, 1, 1, tzinfo=timezone.utc) time2 = datetime.now(timezone(timedelta(hours=8))) if (len(data1min.data) == 0) or \ (Period_Time * 3 < (time2 - time1).total_seconds()): # * 代表非实时数据,已经超出当前时间区间 if ((time2 - time1).total_seconds() > 14400): msg = u"实时行情 MariaDB 数据库已经走丢了,已经超过 %d 小时没有更新,请主人检查 DS1517+ 情况。—— %s" % ( time_window, QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))), ) else: msg = u"huobi.pro 实时行情数据已经超过 %d 秒没有更新,请主人检查那台宇宙超级无敌的主机和海外网络连接情况。—— %s" % ( (time2 - time1).total_seconds(), QA_util_timestamp_to_str( datetime.now(timezone(timedelta(hours=8)))), ) if ((QA_util_datetime_to_Unix_timestamp( datetime.now(timezone(timedelta(hours=8)))) - stock_conf['alert_counter']) >= 300): stock_conf['alert_counter'] = QA_util_datetime_to_Unix_timestamp( datetime.now(timezone(timedelta(hours=8)))) send_servercha(msg) #send_wechat(msg) print(msg) return stock_conf
def QA_util_find_missing_kline(symbol, freq, start_epoch=datetime(2017, 10, 1, tzinfo=tzutc()), tzlocalize='Asia/Shanghai'): """ 查找24小时不间断的连续交易市场中缺失的 kline 历史数据,生成缺失历史数据时间段 """ FREQUENCE_PERIOD_TIME = { FREQUENCE.ONE_MIN: 60, FREQUENCE.FIVE_MIN: 300, FREQUENCE.FIFTEEN_MIN: 900, FREQUENCE.THIRTY_MIN: 1800, FREQUENCE.SIXTY_MIN: 3600, FREQUENCE.HOUR: 3600, FREQUENCE.DAY: 86400, } if (freq != FREQUENCE.DAY): col = DATABASE.cryptocurrency_min col.create_index([("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) col.create_index([("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)]) col.create_index([('time_stamp', pymongo.ASCENDING)]) # 查询历史数据 query_id = {"symbol": symbol, 'type': freq} refcount = col.count_documents(query_id) _data = [] cursor = col.find(query_id).sort('time_stamp', 1) for item in cursor: _data.append([ str(item['symbol']), item['time_stamp'], item['date'], item['datetime'], item['type'] ]) _data = pd.DataFrame( _data, columns=['symbol', 'time_stamp', 'date', 'datetime', 'type']) _data = _data.set_index(pd.DatetimeIndex(_data['datetime']), drop=False) else: col = DATABASE.cryptocurrency_day col.create_index([("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) # 查询是否新 tick query_id = {"symbol": symbol} refcount = col.count_documents(query_id) cursor = col.find(query_id).sort('time_stamp', 1) _data = [] for item in cursor: _data.append([ str(item['symbol']), item['time_stamp'], item['date'], item['datetime'] ]) _data = pd.DataFrame( _data, columns=['symbol', 'time_stamp', 'date', 'datetime']).drop_duplicates() _data['date'] = pd.to_datetime(_data['date'], utc=False) _data = _data.set_index(pd.DatetimeIndex(_data['date']), drop=False) if (freq != FREQUENCE.DAY): # cryptocurrency_min 中的 Date/Datetime 字段均为北京时间 leak_datetime = pd.date_range(_data.index.min(), _data.index.max(), freq=freq).difference( _data.index).tz_localize(tzlocalize) if (int(_data.iloc[0].time_stamp) > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol, freq, QA_util_print_timestamp(_data.iloc[0].time_stamp), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) else: leak_datetime = pd.date_range(_data.index.min(), _data.index.max(), freq='1D').difference( _data.index).tz_localize(tzlocalize) if (int(time.mktime(start_epoch.utctimetuple())) > int( _data.iloc[0].time_stamp)): miss_kline = pd.DataFrame(columns=['expected', 'between', 'missing']) else: miss_kline = pd.DataFrame([[ int(time.mktime(start_epoch.utctimetuple())), int(_data.iloc[0].time_stamp), '{} to {}'.format( start_epoch, _data.iloc[0].date) ]], columns=['expected', 'between', 'missing']) expected = None for x in range(0, len(leak_datetime)): if (expected is None): expected = int(leak_datetime[x].timestamp()) if ((expected is not None) and (x > 1) and (int(leak_datetime[x].timestamp()) != int(leak_datetime[x - 1].timestamp() + FREQUENCE_PERIOD_TIME[freq]))) or \ ((expected is not None) and (x > 1) and (x == len(leak_datetime) - 1)): between = int(leak_datetime[x - 1].timestamp() + FREQUENCE_PERIOD_TIME[freq]) miss_kline = miss_kline.append( { 'expected': int(expected), 'between': int(between), 'missing': '{} to {}'.format( pd.to_datetime(expected, unit='s').tz_localize('Asia/Shanghai'), pd.to_datetime(between, unit='s').tz_localize('Asia/Shanghai')) }, ignore_index=True) expected = int(leak_datetime[x].timestamp()) if (int(_data.iloc[-1].time_stamp) + 1 < int( QA_util_datetime_to_Unix_timestamp())): miss_kline = miss_kline.append( { 'expected': int(_data.iloc[-1].time_stamp) + 1, 'between': int(QA_util_datetime_to_Unix_timestamp()), 'missing': '{} to {}'.format( int(_data.iloc[0].time_stamp) + 1, QA_util_datetime_to_Unix_timestamp()) }, ignore_index=True) miss_kline.sort_values(by='expected', ascending=True, inplace=True) if (len(miss_kline) > 0): if (miss_kline.iloc[0].expected > QA_util_datetime_to_Unix_timestamp()) and \ (miss_kline.iloc[0].between > QA_util_datetime_to_Unix_timestamp()): miss_kline.drop(miss_kline.iloc[0], inplace=True) return miss_kline.values
def QA_SU_save_okex_min(frequency='60', ui_log=None, ui_progress=None): """ Save OKEx min kline 分钟线数据,统一转化字段保存数据为 crypto_asset_min """ market = 'okex' symbol_list = QA_fetch_crypto_asset_list(market='okex') col = DATABASE.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info('Starting DOWNLOAD PROGRESS of min Klines from OKEx... ', ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] # 上架仅处理交易对 QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'], 'type': OKEx2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], market='okex')[::-1] else: start_time = OKEx_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ QA_util_datetime_to_Unix_timestamp(start_time), QA_util_datetime_to_Unix_timestamp(end), '{} 到 {}'.format( start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = missing_data_list[i][expected] reqParams['to'] = missing_data_list[i][between] if (reqParams['to'] > (QA_util_datetime_to_Unix_timestamp() + 3600)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_info['symbol'], frequency, QA_util_print_timestamp(reqParams['to']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_okex_kline_min( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_func=QA_SU_save_data_okex_callback) if data is None: QA_util_log_info( 'SYMBOL "{}" from {} to {} has no MORE data'.format( symbol_info['symbol'], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info('DOWNLOAD PROGRESS of min Klines from OKEx accomplished.', ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_bitfinex_day(frequency, ui_log=None, ui_progress=None): """ Save Bitfinex day kline 日线数据,统一转化字段保存数据为 crypto_asset_day """ symbol_template = Bitfinex_SYMBOL symbol_list = QA_fetch_cryptocurrency_list(Bitfinex_EXCHANGE) col = DATABASE.cryptocurrency_day col.create_index([("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format( Bitfinex_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_template.format(symbol_info['symbol']), index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_template.format(symbol_info['symbol']), } ref = col.find(query_id).sort('date_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['date_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], )[::-1] else: start_time = Bitfinex_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ int(QA_util_datetime_to_Unix_timestamp(start_time)), int(QA_util_datetime_to_Unix_timestamp(end)), '{} to {}'.format(start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = int(missing_data_list[i][expected]) reqParams['to'] = int(missing_data_list[i][between]) if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) # 跳到下一个时间段 continue QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_bitfinex_kline( symbol_info['symbol'], time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency, callback_func=QA_SU_save_data_bitfinex_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_template.format(symbol_info['symbol']), QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) continue QA_util_log_info( 'DOWNLOAD PROGRESS of day Klines from Bitfinex accomplished.', ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_bitmex_day(frequency='1d', ui_log=None, ui_progress=None): """ 获取 bitmex K线 日线数据,统一转化字段保存数据为 crypto_asset_day """ symbol_template = bitmex_SYMBOL symbol_list = QA_fetch_cryptocurrency_list(bitmex_EXCHANGE) col = DATABASE.cryptocurrency_day col.create_index([("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format( bitmex_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_template.format(symbol_info['symbol']), index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_template.format(symbol_info['symbol']), } ref = col.find(query_id).sort('date_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['date_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) else: start_time = symbol_info.get('listing', "2018-01-01T00:00:00Z") start_time = parse(start_time) QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) data = QA_fetch_bitmex_kline( symbol_info['symbol'], QA_util_datetime_to_Unix_timestamp(start_time), QA_util_datetime_to_Unix_timestamp(end), frequency, callback_func=QA_SU_save_data_bitmex_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_template.format(symbol_info['symbol']), QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) continue QA_util_log_info( 'DOWNLOAD PROGRESS of day Klines from {:s} accomplished.'.format( bitmex_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress)
def run(initalParams): """ 运行于多线程的ws 请求发送,每3秒请求一次K线图数据,一次请求240条(的时间段),从参数设定的开始时间,请求到终止时间段为止 """ # missing data key indexes expected = 0 between = 1 missing = 2 requested_counter = 1 reqParams = { 'req': initalParams['req'], } # 采用时间倒序抓取,huobi WebSocket api 非常有个性,基本上遵循2个规则: # 'from' 'to' 范围内没交易数据(交易对未上架)——返回 0 数据, # 'from' 'to' 范围内超过300个bar,——返回 0 数据,一个交易对最早是什么时候上架的——无法查询到 # 所以选择倒序算法,从最近的时间开始补历史数据,'from' 'to' 请求范围一直向前递减到请求不到数据为止。 missing_data_list = initalParams['missing'][::-1] for i in range(len(missing_data_list)): reqParams['from'] = int(missing_data_list[i][between] - initalParams['shifting_time']) reqParams['to'] = int(missing_data_list[i][between]) if (reqParams['to'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( initalParams['req'], QA_util_print_timestamp(reqParams['to']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) QA_util_log_info('Fetch %s missing kline:%s 到 %s' % (initalParams['req'], QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) while (reqParams['to'] > missing_data_list[i][expected]): if (self.__batchReqJobs[initalParams['req']].Status == QA_Fetch_Job_Status.FINISHED): # 抓取已经结束了 return True if (self.__batchReqJobs[initalParams['req']].Status == QA_Fetch_Job_Status.READY): reqParams['id'] = "%s_#%d" % (initalParams['id'], requested_counter) if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s} at {:s} but current time is {}' .format( initalParams['req'], QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) self.__batchReqJobs[ initalParams['req']].Request.append(reqParams) self.send_message( reqParams, 'request kline {:s} part#{:d} {:s} to {:s}'.format( initalParams['req'], requested_counter, QA_util_timestamp_to_str( reqParams['from'])[2:16], QA_util_timestamp_to_str( reqParams['to'])[2:16])) # 等待3秒,请求下一个时间段的批量K线数据 reqParams['to'] = int(reqParams['from'] - 1) reqParams['from'] = int(reqParams['from'] - initalParams['shifting_time']) requested_counter = requested_counter + 1 # 锁定线程,等待回复,避免快速频繁重复请求,会被ban IP的 self.__batchReqJobs[initalParams['req']].setStatus( QA_Fetch_Job_Status.RUNNING) else: # WebSocket请求发出后等待没有回复无需特别处理,一般是SSR/SSL断线,会自动重连,继续补缺失数据 time.sleep(10) self.send_message( reqParams, 'request kline {:s} part#{:d} {:s} to {:s}'.format( initalParams['req'], requested_counter, QA_util_timestamp_to_str( reqParams['from'])[2:16], QA_util_timestamp_to_str( reqParams['to'])[2:16])) pass time.sleep(3)
def QA_SU_save_huobi_min( frequency=CandlestickInterval.MIN1, fetch_range='all', ui_log=None, ui_progress=None, ): """ 下载火币K线分钟数据,统一转化字段保存数据为 cryptocurrency_min """ symbol_template = huobi_SYMBOL symbol_list = QA_fetch_cryptocurrency_list(huobi_EXCHANGE) print("XXX", huobi_EXCHANGE, symbol_list) col = DATABASE.cryptocurrency_min col.create_index([("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of min Klines from {:s}... '.format( huobi_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] if ((fetch_range != 'all') and (symbol_info['symbol'] not in fetch_range)): # Process save_range[] only continue QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_template.format(symbol_info['symbol']), index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_template.format(symbol_info['symbol']), 'type': Huobi2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], )[::-1] else: start_time = huobi_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ int(QA_util_datetime_to_Unix_timestamp(start_time)), int(QA_util_datetime_to_Unix_timestamp(end)), '{} to {}'.format(start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = int(missing_data_list[i][expected]) reqParams['to'] = int(missing_data_list[i][between]) if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) # 跳到下一个时间段 continue QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_huobi_kline_subscription( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_func=QA_SU_save_data_huobi_callback) if data is None: QA_util_log_info( 'SYMBOL "{}" from {} to {} has no MORE data'.format( symbol_template.format(symbol_info['symbol']), QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info( 'DOWNLOAD PROGRESS of min Klines from {:s} accomplished.'.format( huobi_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_huobi_min(client=DATABASE, ui_log=None, ui_progress=None, frequency=CandlestickInterval.MIN1, fetch_range='all'): """ 下载火币K线分钟数据,统一转化字段保存数据为 crypto_asset_min """ def QA_SU_save_data_huobi_min_callback(data, symbol=''): """ 异步获取数据回调用的 MongoDB 存储函数 """ market = 'huobi' QA_util_log_info( 'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[0]['time_stamp'])), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[-1]['time_stamp'])), len(data)), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol, 'market': market, 'type': Huobi2QA_FREQUENCY_DICT[frequency], 'time_stamp': { '$in': list(map(lambda x: x['time_stamp'], data)) } } if (col.count_documents(query_id) > 0): # 删掉重复数据 col.delete_many(query_id) try: col.insert_many(data) except: QA_util_log_expection( 'QA_SU_save_huobi_day():Insert_many(kline) to {} got Exception {}' .format(symbol_info['symbol'], len(data))) pass symbol_list = QA_fetch_crypto_asset_list('huobi') col = client.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of min Klines from huobi.pro... ') for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] if ((fetch_range != 'all') and (symbol_info['symbol'] not in fetch_range)): # Process save_range[] only continue QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'], 'type': Huobi2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency])[::-1] if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = missing_data_list[i][expected] reqParams['to'] = missing_data_list[i][between] if (reqParams['to'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s}({:d}) but current time is {}' .format(symbol_info['symbol'], frequency, missing_data_list[i][missing], missing_data_list[i][between], QA_util_datetime_to_Unix_timestamp())) QA_util_log_info( 'Fetch %s missing %s kline:%s 到 %s' % (symbol_info['symbol'], frequency, QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) QA_fetch_huobi_kline_subscription( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_save_data_func=QA_SU_save_data_huobi_callback) else: start_time = huobi_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) data = QA_fetch_huobi_kline( symbol_info['symbol'], time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency=frequency, callback_save_data_func=QA_SU_save_data_huobi_min_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info( 'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[0]['time_stamp'])), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[-1]['time_stamp'])), len(data))) QA_util_log_info( 'DOWNLOAD PROGRESS of min Klines from huobi.pro accomplished.')
def QA_fetch_huobi_kline_subscription( symbol, start_time, end_time, frequency, callback_func ): """ Get the symbol‘s candlestick data by subscription """ reqParams = {} reqParams['from'] = int(end_time - FREQUENCY_SHIFTING[frequency]) reqParams['to'] = int(end_time) requested_counter = 1 sub_client = QA_Fetch_Huobi( callback_save_data_func=callback_func, find_missing_kline_func=QA_util_find_missing_kline ) datas = list() retries = 1 while (reqParams['to'] > start_time): if ((reqParams['from'] > QA_util_datetime_to_Unix_timestamp())) or \ ((reqParams['from'] > reqParams['to'])): # 出现“未来”时间,一般是默认时区设置,或者时间窗口滚动前移错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol, frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp() ) ) ) # 跳到下一个时间段 reqParams['to'] = int(reqParams['from'] - 1) reqParams['from'] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency]) continue retries = 1 frame = None while (retries != 0): try: frame = sub_client.run_request_historical_kline( symbol, frequency, reqParams['from'], reqParams['to'], requested_counter ) if (frame is None): # 火币网的 WebSocket 接口机制很奇特,返回len(data)==0 # 就说明已经超越这个交易对的上架时间,不再有更多数据了。 # 当前 Symbol Klines 抓取已经结束了 return None else: retries = 0 except Exception: retries = retries + 1 if (retries % 6 == 0): print(ILOVECHINA) print("Retry request_historical_kline #{}".format(retries - 1)) time.sleep(0.5) if (retries == 0): # 等待3秒,请求下一个时间段的批量K线数据 reqParams['to'] = int(reqParams['from'] - 1) reqParams['from' ] = int(reqParams['from'] - FREQUENCY_SHIFTING[frequency]) requested_counter = requested_counter + 1 # 这一步冗余,如果是开启实时抓取会自动被 WebSocket.on_messgae_callback 事件处理函数保存, # 但不确定不开启实时行情抓取会不会绑定on_messgae事件,所以保留冗余。 callback_func(data=frame, freq=frequency) time.sleep(0.5) if ((retries == 0) and (len(frame) > 0)): return frame else: return None