def QA_SU_save_okex_min(frequency='60', ui_log=None, ui_progress=None): """ Save OKEx min kline 分钟线数据,统一转化字段保存数据为 crypto_asset_min """ market = 'okex' symbol_list = QA_fetch_crypto_asset_list(market='okex') col = DATABASE.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info('Starting DOWNLOAD PROGRESS of min Klines from OKEx... ', ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] # 上架仅处理交易对 QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'], 'type': OKEx2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], market='okex')[::-1] else: start_time = OKEx_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ QA_util_datetime_to_Unix_timestamp(start_time), QA_util_datetime_to_Unix_timestamp(end), '{} 到 {}'.format( start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = missing_data_list[i][expected] reqParams['to'] = missing_data_list[i][between] if (reqParams['to'] > (QA_util_datetime_to_Unix_timestamp() + 3600)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_info['symbol'], frequency, QA_util_print_timestamp(reqParams['to']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_info['symbol'], OKEx2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_okex_kline_min( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_func=QA_SU_save_data_okex_callback) if data is None: QA_util_log_info( 'SYMBOL "{}" from {} to {} has no MORE data'.format( symbol_info['symbol'], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info('DOWNLOAD PROGRESS of min Klines from OKEx accomplished.', ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_huobi_min( frequency=CandlestickInterval.MIN1, fetch_range='all', ui_log=None, ui_progress=None, ): """ 下载火币K线分钟数据,统一转化字段保存数据为 cryptocurrency_min """ symbol_template = huobi_SYMBOL symbol_list = QA_fetch_cryptocurrency_list(huobi_EXCHANGE) print("XXX", huobi_EXCHANGE, symbol_list) col = DATABASE.cryptocurrency_min col.create_index([("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of min Klines from {:s}... '.format( huobi_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] if ((fetch_range != 'all') and (symbol_info['symbol'] not in fetch_range)): # Process save_range[] only continue QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_template.format(symbol_info['symbol']), index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_template.format(symbol_info['symbol']), 'type': Huobi2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], )[::-1] else: start_time = huobi_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ int(QA_util_datetime_to_Unix_timestamp(start_time)), int(QA_util_datetime_to_Unix_timestamp(end)), '{} to {}'.format(start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = int(missing_data_list[i][expected]) reqParams['to'] = int(missing_data_list[i][between]) if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) # 跳到下一个时间段 continue QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_huobi_kline_subscription( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_func=QA_SU_save_data_huobi_callback) if data is None: QA_util_log_info( 'SYMBOL "{}" from {} to {} has no MORE data'.format( symbol_template.format(symbol_info['symbol']), QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info( 'DOWNLOAD PROGRESS of min Klines from {:s} accomplished.'.format( huobi_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_bitfinex_day(frequency, ui_log=None, ui_progress=None): """ Save Bitfinex day kline 日线数据,统一转化字段保存数据为 crypto_asset_day """ symbol_template = Bitfinex_SYMBOL symbol_list = QA_fetch_cryptocurrency_list(Bitfinex_EXCHANGE) col = DATABASE.cryptocurrency_day col.create_index([("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of day Klines from {:s}... '.format( Bitfinex_EXCHANGE), ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_template.format(symbol_info['symbol']), index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_template.format(symbol_info['symbol']), } ref = col.find(query_id).sort('date_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['date_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], )[::-1] else: start_time = Bitfinex_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ int(QA_util_datetime_to_Unix_timestamp(start_time)), int(QA_util_datetime_to_Unix_timestamp(end)), '{} to {}'.format(start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = int(missing_data_list[i][expected]) reqParams['to'] = int(missing_data_list[i][between]) if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 QA_util_log_info( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_template.format(symbol_info['symbol']), frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) # 跳到下一个时间段 continue QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_template.format(symbol_info['symbol']), Bitfinex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_bitfinex_kline( symbol_info['symbol'], time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency, callback_func=QA_SU_save_data_bitfinex_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_template.format(symbol_info['symbol']), QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) continue QA_util_log_info( 'DOWNLOAD PROGRESS of day Klines from Bitfinex accomplished.', ui_log=ui_log, ui_progress=ui_progress)
def QA_SU_save_huobi_min(client=DATABASE, ui_log=None, ui_progress=None, frequency=CandlestickInterval.MIN1, fetch_range='all'): """ 下载火币K线分钟数据,统一转化字段保存数据为 crypto_asset_min """ def QA_SU_save_data_huobi_min_callback(data, symbol=''): """ 异步获取数据回调用的 MongoDB 存储函数 """ market = 'huobi' QA_util_log_info( 'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[0]['time_stamp'])), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[-1]['time_stamp'])), len(data)), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol, 'market': market, 'type': Huobi2QA_FREQUENCY_DICT[frequency], 'time_stamp': { '$in': list(map(lambda x: x['time_stamp'], data)) } } if (col.count_documents(query_id) > 0): # 删掉重复数据 col.delete_many(query_id) try: col.insert_many(data) except: QA_util_log_expection( 'QA_SU_save_huobi_day():Insert_many(kline) to {} got Exception {}' .format(symbol_info['symbol'], len(data))) pass symbol_list = QA_fetch_crypto_asset_list('huobi') col = client.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of min Klines from huobi.pro... ') for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] if ((fetch_range != 'all') and (symbol_info['symbol'] not in fetch_range)): # Process save_range[] only continue QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'], 'type': Huobi2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency])[::-1] if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = missing_data_list[i][expected] reqParams['to'] = missing_data_list[i][between] if (reqParams['to'] > (QA_util_datetime_to_Unix_timestamp() + 120)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s}({:d}) but current time is {}' .format(symbol_info['symbol'], frequency, missing_data_list[i][missing], missing_data_list[i][between], QA_util_datetime_to_Unix_timestamp())) QA_util_log_info( 'Fetch %s missing %s kline:%s 到 %s' % (symbol_info['symbol'], frequency, QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) QA_fetch_huobi_kline_subscription( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_save_data_func=QA_SU_save_data_huobi_callback) else: start_time = huobi_MIN_DATE QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) data = QA_fetch_huobi_kline( symbol_info['symbol'], time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency=frequency, callback_save_data_func=QA_SU_save_data_huobi_min_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info( 'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format( symbol_info['symbol'], Huobi2QA_FREQUENCY_DICT[frequency], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[0]['time_stamp'])), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data[-1]['time_stamp'])), len(data))) QA_util_log_info( 'DOWNLOAD PROGRESS of min Klines from huobi.pro accomplished.')