def QA_SU_save_data_huobi_callback(data, freq): """ 异步获取数据回调用的 MongoDB 存储函数 """ if (freq not in [FREQUENCE.DAY, CandlestickInterval.DAY1, '1d']): col = DATABASE.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) # 查询是否新 tick query_id = { "symbol": data.iloc[0].symbol, 'market': data.iloc[0].market, 'type': data.iloc[0].type, 'time_stamp': { '$in': data['time_stamp'].tolist() } } refcount = col.count_documents(query_id) else: col = DATABASE.crypto_asset_day col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) # 查询是否新 tick query_id = { "symbol": data.iloc[0].symbol, 'market': data.iloc[0].market, 'date_stamp': { '$in': data['date_stamp'].tolist() } } refcount = col.count_documents(query_id) if refcount > 0: if (len(data) > 1): # 删掉重复数据 col.delete_many(query_id) data = QA_util_to_json_from_pandas(data) col.insert_many(data) else: # 持续接收行情,更新记录 data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) col.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) col.insert_many(data)
def save_top10deal(top10deal_df): """保存东方财富--北向十大成交记录 (基础函数) """ assert top10deal_df is not None, 'top10deal_df must be' assert len(top10deal_df) > 0, 'top10deal_df must not be 0 row' assert len(top10deal_df['type'].unique() ) == 1, 'top10deal_df[type] must be unique in once task' data = top10deal_df.reset_index() freq = data.iloc[0].type coll = DATABASE.index_north_em_10top coll.create_index([("date_stamp", pymongo.ASCENDING)], unique=False) coll.create_index([('type', pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=False) coll.create_index([('model', pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=False) # 查询是否新数据 query_id = { 'type': data.iloc[0].type, 'date_stamp': { '$in': data['date_stamp'].tolist() } } refcount = coll.count_documents(query_id) try: if refcount > 0: if (len(data) > 1): # 删掉重复数据 coll.delete_many(query_id) data = QA_util_to_json_from_pandas(data) coll.insert_many(data) else: # 持续接收行情,更新记录 if ('created_at' in data.columns): data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) coll.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) coll.insert_many(data) except Exception as e: if (data is not None): traceback.print_exception(type(e), e, sys.exc_info()[2]) print(u'save_top10deal failed!\n', e)
def QA_SU_save_data_okex_callback(data, freq): """ 异步获取数据回调用的 MongoDB 存储函数 """ QA_util_log_info( 'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format( data.iloc[0].symbol, freq, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.iloc[0].time_stamp))[2:16], time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(data.iloc[-1].time_stamp))[2:16], len(data))) if (freq not in ['1day', '86400', 'day', '1d']): col = DATABASE.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) # 查询是否新 tick query_id = { "symbol": data.iloc[0].symbol, 'market': data.iloc[0].market, 'type': data.iloc[0].type, 'time_stamp': { '$in': data['time_stamp'].tolist() } } refcount = col.count_documents(query_id) else: col = DATABASE.crypto_asset_day col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) # 查询是否新 tick query_id = { "symbol": data.iloc[0].symbol, 'market': data.iloc[0].market, 'date_stamp': { '$in': data['date_stamp'].tolist() } } refcount = col.count_documents(query_id) if refcount > 0: if (len(data) > 1): # 删掉重复数据 col.delete_many(query_id) data = QA_util_to_json_from_pandas(data) col.insert_many(data) else: # 持续接收行情,更新记录 data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) col.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) col.insert_many(data)
def GQSignal_util_save_indices_min(code, indices, frequence, market_type=QA.MARKET_TYPE.STOCK_CN, portfolio='myportfolio', ui_log=None, ui_progress=None): """ 在数据库中保存所有计算出来的指标信息,用于汇总评估和筛选数据——分钟线 save stock_indices, state Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ def _check_index(coll_indices): coll_indices.create_index([("code", pymongo.ASCENDING), ("type", pymongo.ASCENDING), (FLD.DATETIME, pymongo.ASCENDING), ], unique=True) coll_indices.create_index([("code", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING), ], unique=True) coll_indices.create_index([(FLD.DATETIME, pymongo.ASCENDING), ("type", pymongo.ASCENDING), (ST.TRIGGER_R5, pymongo.ASCENDING), ], ) coll_indices.create_index([(FLD.DATETIME, pymongo.ASCENDING), ("type", pymongo.ASCENDING), (ST.POSITION_R5, pymongo.ASCENDING), ], ) coll_indices.create_index([("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING), (ST.TRIGGER_R5, pymongo.ASCENDING), ], ) coll_indices.create_index([("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING), (ST.POSITION_R5, pymongo.ASCENDING), ], ) coll_indices.create_index([(FLD.DATETIME, pymongo.ASCENDING), ("type", pymongo.ASCENDING), (FLD.FLU_POSITIVE, pymongo.ASCENDING), ], ) coll_indices.create_index([("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING), (FLD.FLU_POSITIVE, pymongo.ASCENDING), ], ) coll_indices.create_index([("code", pymongo.ASCENDING), ("type", pymongo.ASCENDING), (FLD.DATETIME, pymongo.ASCENDING), (ST.CANDIDATE, pymongo.ASCENDING), ], unique=True) coll_indices.create_index([("code", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING), (ST.CANDIDATE, pymongo.ASCENDING), ], unique=True) def _formatter_data(indices, frequence): frame = indices.reset_index(1, drop=False) # UTC时间转换为北京时间 frame['date'] = pd.to_datetime(frame.index, ).tz_localize('Asia/Shanghai') frame['date'] = frame['date'].dt.strftime('%Y-%m-%d') frame['datetime'] = pd.to_datetime(frame.index, ).tz_localize('Asia/Shanghai') frame['datetime'] = frame['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S') # GMT+0 String 转换为 UTC Timestamp frame['time_stamp'] = pd.to_datetime(frame['datetime']).astype(np.int64) // 10 ** 9 frame['type'] = frequence frame['created_at'] = int(time.mktime(datetime.datetime.now().utctimetuple())) frame = frame.tail(len(frame) - 150) return frame client = QASETTING.client[AKA.SYSTEM_NAME] # 同时写入横表和纵表,减少查询困扰 # coll_day = client.get_collection( # 'indices_{}'.format(datetime.date.today())) try: if (market_type == QA.MARKET_TYPE.STOCK_CN): # coll_indices = client.stock_cn_indices_min coll_indices = client.get_collection('stock_cn_indices_{}'.format(portfolio)) elif (market_type == QA.MARKET_TYPE.INDEX_CN): # coll_indices = client.index_cn_indices_min coll_indices = client.get_collection('index_cn_indices_{}'.format(portfolio)) elif (market_type == QA.MARKET_TYPE.FUND_CN): # coll_indices = client.future_cn_indices_min coll_indices = client.get_collection('fund_cn_indices_{}'.format(portfolio)) elif (market_type == QA.MARKET_TYPE.FUTURE_CN): # coll_indices = client.future_cn_indices_min coll_indices = client.get_collection('future_cn_indices_{}'.format(portfolio)) elif (market_type == QA.MARKET_TYPE.CRYPTOCURRENCY): # coll_indices = client.cryptocurrency_indices_min coll_indices = client.get_collection('cryptocurrency_indices_{}'.format(portfolio)) else: QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log) return False except Exception as e: QA_util_log_info(e) QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log) return False _check_index(coll_indices) data = _formatter_data(indices, frequence) err = [] # 查询是否新 tick query_id = { "code": code, 'type': frequence, "time_stamp": { '$in': data['time_stamp'].tolist() } } refcount = coll_indices.count_documents(query_id) if refcount > 0: if (len(data) > 1): # 删掉重复数据 coll_indices.delete_many(query_id) data = QA_util_to_json_from_pandas(data) coll_indices.insert_many(data) else: # 持续更新模式,更新单条记录 data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) coll_indices.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) coll_indices.insert_many(data) return True
def save_north_line(north_df): """保存东方财富--北向数据 (基础函数) """ assert north_df is not None, 'north_df must be' assert len(north_df) > 0, 'north_df must not be 0 row' data = north_df.reset_index() freq = data.iloc[0].type if (freq == QA.FREQUENCE.DAY): coll = DATABASE.index_north_em_day coll.create_index([('code', pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) elif (freq == QA.FREQUENCE.ONE_MIN): coll = DATABASE.tmp_1min_index_north_em coll.create_index([('code', pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING)], unique=True) else: raise Error('save_north_line: freq type error') # 查询是否新数据 if (freq == QA.FREQUENCE.DAY): query_id = { 'code': data.iloc[0].code, 'date_stamp': { '$in': data['date_stamp'].tolist() } } else: query_id = { 'code': data.iloc[0].code, 'time_stamp': { '$in': data['time_stamp'].tolist() } } refcount = coll.count_documents(query_id) try: if refcount > 0: if (len(data) > 1): # 删掉重复数据 coll.delete_many(query_id) data = QA_util_to_json_from_pandas(data) coll.insert_many(data) else: # 持续接收行情,更新记录 if ('created_at' in data.columns): data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) coll.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) coll.insert_many(data) except Exception as e: if (data is not None): traceback.print_exception(type(e), e, sys.exc_info()[2]) print(u'save_north_line failed!\n', e)
def GQ_save_test(code, indices, market_type=QA.MARKET_TYPE.STOCK_CN, portfolio='myportfolio', ui_log=None, ui_progress=None): """ 在数据库中保存所有计算出来的股票日线指标,用于汇总评估和筛选数据——日线 save stock_indices, state Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ def _check_index(coll_indices): coll_indices.create_index([ ("code", pymongo.ASCENDING), ('datetime', pymongo.ASCENDING), ], unique=True) coll_indices.create_index([ ("date", pymongo.ASCENDING), ('TRG_R5', pymongo.ASCENDING), ], ) def _formatter_data(indices): frame = indices.reset_index(1, drop=False) # UTC时间转换为北京时间 frame['date'] = pd.to_datetime( frame.index, ).tz_localize('Asia/Shanghai') frame['date'] = frame['date'].dt.strftime('%Y-%m-%d') frame['datetime'] = pd.to_datetime( frame.index, ).tz_localize('Asia/Shanghai') frame['datetime'] = frame['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S') # GMT+0 String 转换为 UTC Timestamp frame['date_stamp'] = pd.to_datetime(frame['date']).astype( np.int64) // 10**9 frame['created_at'] = int( time.mktime(datetime.datetime.now().utctimetuple())) frame = frame.tail(len(frame) - 150) return frame client = QASETTING.client['quantaxis'] # 同时写入横表和纵表,减少查询困扰 #coll_day = client.get_collection( # 'indices_{}'.format(datetime.date.today())) try: if (market_type == QA.MARKET_TYPE.STOCK_CN): #coll_indices = client.stock_cn_indices_day coll_indices = client.get_collection( 'stock_cn_indices_{}'.format(portfolio)) elif (market_type == QA.MARKET_TYPE.INDEX_CN): #coll_indices = client.index_cn_indices_day coll_indices = client.get_collection( 'index_cn_indices_{}'.format(portfolio)) else: QA_util_log_info('WTF IS THIS! {} \n '.format(market_type), ui_log=ui_log) return False except Exception as e: QA_util_log_info(e) QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log) return False _check_index(coll_indices) data = _formatter_data(indices) err = [] # 查询是否新 tick query_id = { "code": code, 'date_stamp': { '$in': data['date_stamp'].tolist() } } refcount = coll_indices.count_documents(query_id) if refcount > 0: if (len(data) > 1): # 删掉重复数据 coll_indices.delete_many(query_id) data = QA_util_to_json_from_pandas(data) coll_indices.insert_many(data) else: # 持续更新模式,更新单条记录 data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) coll_indices.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) coll_indices.insert_many(data) return True
def save_stock_block_kline(block_kline_df): """保存东方财富股票概念板块K线数据(功能函数) 注:1min数据日内产生,有可能会有遗漏,当临时数据,故另外存放,随时丢弃。 """ assert block_kline_df is not None, 'block_kline_df must be' assert len(block_kline_df) > 0, 'block_kline_df must not be 0 row' data = block_kline_df.reset_index() freq = data.iloc[0].type assert freq in [ QA.FREQUENCE.DAY, QA.FREQUENCE.HOUR, QA.FREQUENCE.FIVE_MIN, QA.FREQUENCE.ONE_MIN ], 'freq only support DAY|HOUR|FIVE_MIN|ONE_MIN(自我规制)' if (freq == QA.FREQUENCE.DAY): coll = DATABASE.stock_block_em_day coll.create_index([('code', pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) elif (freq == QA.FREQUENCE.ONE_MIN): coll = DATABASE.tmp_1min_stock_block_em coll.create_index([('code', pymongo.ASCENDING), ("date", pymongo.ASCENDING)], unique=False) coll.create_index([('code', pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING)], unique=True) else: coll = DATABASE.stock_block_em_min coll.create_index([('code', pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("date", pymongo.ASCENDING)], unique=False) coll.create_index([('code', pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("time_stamp", pymongo.ASCENDING)], unique=True) # 查询是否新数据 if (freq == QA.FREQUENCE.DAY): query_id = { 'code': data.iloc[0].code, 'date_stamp': { '$in': data['date_stamp'].tolist() } } else: query_id = { 'code': data.iloc[0].code, 'type': freq, 'time_stamp': { '$in': data['time_stamp'].tolist() } } refcount = coll.count_documents(query_id) try: if refcount > 0: if (len(data) > 1): # 删掉重复数据 coll.delete_many(query_id) data = QA_util_to_json_from_pandas(data) coll.insert_many(data) else: # 持续接收行情,更新记录 if ('created_at' in data.columns): data.drop('created_at', axis=1, inplace=True) data = QA_util_to_json_from_pandas(data) coll.replace_one(query_id, data[0]) else: # 新 tick,插入记录 data = QA_util_to_json_from_pandas(data) coll.insert_many(data) except Exception as e: if (data is not None): traceback.print_exception(type(e), e, sys.exc_info()[2]) print(u'save_stock_block_kline failed!!\n', e)