def QA_SU_save_data_huobi_callback(data, freq):
"""
异步获取数据回调用的 MongoDB 存储函数
"""
if (freq not in [FREQUENCE.DAY, CandlestickInterval.DAY1, '1d']):
col = DATABASE.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
# 查询是否新 tick
query_id = {
"symbol": data.iloc[0].symbol,
'market': data.iloc[0].market,
'type': data.iloc[0].type,
'time_stamp': {
'$in': data['time_stamp'].tolist()
}
}
refcount = col.count_documents(query_id)
else:
col = DATABASE.crypto_asset_day
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
# 查询是否新 tick
query_id = {
"symbol": data.iloc[0].symbol,
'market': data.iloc[0].market,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
refcount = col.count_documents(query_id)
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
col.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
col.insert_many(data)
else:
# 持续接收行情，更新记录
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
col.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
col.insert_many(data)
def save_top10deal(top10deal_df):
"""保存东方财富--北向十大成交记录 (基础函数)
"""
assert top10deal_df is not None, 'top10deal_df must be'
assert len(top10deal_df) > 0, 'top10deal_df must not be 0 row'
assert len(top10deal_df['type'].unique()
) == 1, 'top10deal_df[type] must be unique in once task'
data = top10deal_df.reset_index()
freq = data.iloc[0].type
coll = DATABASE.index_north_em_10top
coll.create_index([("date_stamp", pymongo.ASCENDING)], unique=False)
coll.create_index([('type', pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=False)
coll.create_index([('model', pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=False)
# 查询是否新数据
query_id = {
'type': data.iloc[0].type,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
refcount = coll.count_documents(query_id)
try:
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
coll.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
else:
# 持续接收行情，更新记录
if ('created_at' in data.columns):
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
coll.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
except Exception as e:
if (data is not None):
traceback.print_exception(type(e), e, sys.exc_info()[2])
print(u'save_top10deal failed!\n', e)
def QA_SU_save_data_okex_callback(data, freq):
"""
异步获取数据回调用的 MongoDB 存储函数
"""
QA_util_log_info(
'SYMBOL "{}" Recived "{}" from {} to {} in total {} klines'.format(
data.iloc[0].symbol, freq,
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data.iloc[0].time_stamp))[2:16],
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(data.iloc[-1].time_stamp))[2:16],
len(data)))
if (freq not in ['1day', '86400', 'day', '1d']):
col = DATABASE.crypto_asset_min
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING)],
unique=True)
# 查询是否新 tick
query_id = {
"symbol": data.iloc[0].symbol,
'market': data.iloc[0].market,
'type': data.iloc[0].type,
'time_stamp': {
'$in': data['time_stamp'].tolist()
}
}
refcount = col.count_documents(query_id)
else:
col = DATABASE.crypto_asset_day
col.create_index([('market', pymongo.ASCENDING),
("symbol", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
# 查询是否新 tick
query_id = {
"symbol": data.iloc[0].symbol,
'market': data.iloc[0].market,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
refcount = col.count_documents(query_id)
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
col.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
col.insert_many(data)
else:
# 持续接收行情，更新记录
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
col.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
col.insert_many(data)
def GQSignal_util_save_indices_min(code,
indices,
frequence,
market_type=QA.MARKET_TYPE.STOCK_CN,
portfolio='myportfolio',
ui_log=None,
ui_progress=None):
"""
在数据库中保存所有计算出来的指标信息，用于汇总评估和筛选数据——分钟线
save stock_indices, state
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
def _check_index(coll_indices):
coll_indices.create_index([("code",
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
(FLD.DATETIME,
pymongo.ASCENDING), ],
unique=True)
coll_indices.create_index([("code",
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
("time_stamp",
pymongo.ASCENDING), ],
unique=True)
coll_indices.create_index([(FLD.DATETIME,
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
(ST.TRIGGER_R5,
pymongo.ASCENDING), ], )
coll_indices.create_index([(FLD.DATETIME,
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
(ST.POSITION_R5,
pymongo.ASCENDING), ], )
coll_indices.create_index([("type",
pymongo.ASCENDING),
("time_stamp",
pymongo.ASCENDING),
(ST.TRIGGER_R5,
pymongo.ASCENDING), ], )
coll_indices.create_index([("type",
pymongo.ASCENDING),
("time_stamp",
pymongo.ASCENDING),
(ST.POSITION_R5,
pymongo.ASCENDING), ], )
coll_indices.create_index([(FLD.DATETIME,
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
(FLD.FLU_POSITIVE,
pymongo.ASCENDING), ], )
coll_indices.create_index([("type",
pymongo.ASCENDING),
("time_stamp",
pymongo.ASCENDING),
(FLD.FLU_POSITIVE,
pymongo.ASCENDING), ], )
coll_indices.create_index([("code",
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
(FLD.DATETIME,
pymongo.ASCENDING),
(ST.CANDIDATE,
pymongo.ASCENDING), ],
unique=True)
coll_indices.create_index([("code",
pymongo.ASCENDING),
("type",
pymongo.ASCENDING),
("time_stamp",
pymongo.ASCENDING),
(ST.CANDIDATE,
pymongo.ASCENDING), ],
unique=True)
def _formatter_data(indices, frequence):
frame = indices.reset_index(1, drop=False)
# UTC时间转换为北京时间
frame['date'] = pd.to_datetime(frame.index, ).tz_localize('Asia/Shanghai')
frame['date'] = frame['date'].dt.strftime('%Y-%m-%d')
frame['datetime'] = pd.to_datetime(frame.index, ).tz_localize('Asia/Shanghai')
frame['datetime'] = frame['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
# GMT+0 String 转换为 UTC Timestamp
frame['time_stamp'] = pd.to_datetime(frame['datetime']).astype(np.int64) // 10 ** 9
frame['type'] = frequence
frame['created_at'] = int(time.mktime(datetime.datetime.now().utctimetuple()))
frame = frame.tail(len(frame) - 150)
return frame
client = QASETTING.client[AKA.SYSTEM_NAME]
# 同时写入横表和纵表，减少查询困扰
# coll_day = client.get_collection(
# 'indices_{}'.format(datetime.date.today()))
try:
if (market_type == QA.MARKET_TYPE.STOCK_CN):
# coll_indices = client.stock_cn_indices_min
coll_indices = client.get_collection('stock_cn_indices_{}'.format(portfolio))
elif (market_type == QA.MARKET_TYPE.INDEX_CN):
# coll_indices = client.index_cn_indices_min
coll_indices = client.get_collection('index_cn_indices_{}'.format(portfolio))
elif (market_type == QA.MARKET_TYPE.FUND_CN):
# coll_indices = client.future_cn_indices_min
coll_indices = client.get_collection('fund_cn_indices_{}'.format(portfolio))
elif (market_type == QA.MARKET_TYPE.FUTURE_CN):
# coll_indices = client.future_cn_indices_min
coll_indices = client.get_collection('future_cn_indices_{}'.format(portfolio))
elif (market_type == QA.MARKET_TYPE.CRYPTOCURRENCY):
# coll_indices = client.cryptocurrency_indices_min
coll_indices = client.get_collection('cryptocurrency_indices_{}'.format(portfolio))
else:
QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log)
return False
except Exception as e:
QA_util_log_info(e)
QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log)
return False
_check_index(coll_indices)
data = _formatter_data(indices, frequence)
err = []
# 查询是否新 tick
query_id = {
"code": code,
'type': frequence,
"time_stamp": {
'$in': data['time_stamp'].tolist()
}
}
refcount = coll_indices.count_documents(query_id)
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
coll_indices.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
coll_indices.insert_many(data)
else:
# 持续更新模式，更新单条记录
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
coll_indices.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
coll_indices.insert_many(data)
return True
def save_north_line(north_df):
"""保存东方财富--北向数据 (基础函数)
"""
assert north_df is not None, 'north_df must be'
assert len(north_df) > 0, 'north_df must not be 0 row'
data = north_df.reset_index()
freq = data.iloc[0].type
if (freq == QA.FREQUENCE.DAY):
coll = DATABASE.index_north_em_day
coll.create_index([('code', pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
elif (freq == QA.FREQUENCE.ONE_MIN):
coll = DATABASE.tmp_1min_index_north_em
coll.create_index([('code', pymongo.ASCENDING),
("time_stamp", pymongo.ASCENDING)],
unique=True)
else:
raise Error('save_north_line: freq type error')
# 查询是否新数据
if (freq == QA.FREQUENCE.DAY):
query_id = {
'code': data.iloc[0].code,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
else:
query_id = {
'code': data.iloc[0].code,
'time_stamp': {
'$in': data['time_stamp'].tolist()
}
}
refcount = coll.count_documents(query_id)
try:
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
coll.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
else:
# 持续接收行情，更新记录
if ('created_at' in data.columns):
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
coll.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
except Exception as e:
if (data is not None):
traceback.print_exception(type(e), e, sys.exc_info()[2])
print(u'save_north_line failed!\n', e)
def GQ_save_test(code,
indices,
market_type=QA.MARKET_TYPE.STOCK_CN,
portfolio='myportfolio',
ui_log=None,
ui_progress=None):
"""
在数据库中保存所有计算出来的股票日线指标，用于汇总评估和筛选数据——日线
save stock_indices, state
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
def _check_index(coll_indices):
coll_indices.create_index([
("code", pymongo.ASCENDING),
('datetime', pymongo.ASCENDING),
],
unique=True)
coll_indices.create_index([
("date", pymongo.ASCENDING),
('TRG_R5', pymongo.ASCENDING),
], )
def _formatter_data(indices):
frame = indices.reset_index(1, drop=False)
# UTC时间转换为北京时间
frame['date'] = pd.to_datetime(
frame.index, ).tz_localize('Asia/Shanghai')
frame['date'] = frame['date'].dt.strftime('%Y-%m-%d')
frame['datetime'] = pd.to_datetime(
frame.index, ).tz_localize('Asia/Shanghai')
frame['datetime'] = frame['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
# GMT+0 String 转换为 UTC Timestamp
frame['date_stamp'] = pd.to_datetime(frame['date']).astype(
np.int64) // 10**9
frame['created_at'] = int(
time.mktime(datetime.datetime.now().utctimetuple()))
frame = frame.tail(len(frame) - 150)
return frame
client = QASETTING.client['quantaxis']
# 同时写入横表和纵表，减少查询困扰
#coll_day = client.get_collection(
# 'indices_{}'.format(datetime.date.today()))
try:
if (market_type == QA.MARKET_TYPE.STOCK_CN):
#coll_indices = client.stock_cn_indices_day
coll_indices = client.get_collection(
'stock_cn_indices_{}'.format(portfolio))
elif (market_type == QA.MARKET_TYPE.INDEX_CN):
#coll_indices = client.index_cn_indices_day
coll_indices = client.get_collection(
'index_cn_indices_{}'.format(portfolio))
else:
QA_util_log_info('WTF IS THIS! {} \n '.format(market_type),
ui_log=ui_log)
return False
except Exception as e:
QA_util_log_info(e)
QA_util_log_info('WTF IS THIS! \n ', ui_log=ui_log)
return False
_check_index(coll_indices)
data = _formatter_data(indices)
err = []
# 查询是否新 tick
query_id = {
"code": code,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
refcount = coll_indices.count_documents(query_id)
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
coll_indices.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
coll_indices.insert_many(data)
else:
# 持续更新模式，更新单条记录
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
coll_indices.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
coll_indices.insert_many(data)
return True
def save_stock_block_kline(block_kline_df):
"""保存东方财富股票概念板块K线数据（功能函数）
注：1min数据日内产生，有可能会有遗漏，当临时数据，故另外存放，随时丢弃。
"""
assert block_kline_df is not None, 'block_kline_df must be'
assert len(block_kline_df) > 0, 'block_kline_df must not be 0 row'
data = block_kline_df.reset_index()
freq = data.iloc[0].type
assert freq in [
QA.FREQUENCE.DAY, QA.FREQUENCE.HOUR, QA.FREQUENCE.FIVE_MIN,
QA.FREQUENCE.ONE_MIN
], 'freq only support DAY|HOUR|FIVE_MIN|ONE_MIN（自我规制）'
if (freq == QA.FREQUENCE.DAY):
coll = DATABASE.stock_block_em_day
coll.create_index([('code', pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)],
unique=True)
elif (freq == QA.FREQUENCE.ONE_MIN):
coll = DATABASE.tmp_1min_stock_block_em
coll.create_index([('code', pymongo.ASCENDING),
("date", pymongo.ASCENDING)],
unique=False)
coll.create_index([('code', pymongo.ASCENDING),
("time_stamp", pymongo.ASCENDING)],
unique=True)
else:
coll = DATABASE.stock_block_em_min
coll.create_index([('code', pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("date", pymongo.ASCENDING)],
unique=False)
coll.create_index([('code', pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("time_stamp", pymongo.ASCENDING)],
unique=True)
# 查询是否新数据
if (freq == QA.FREQUENCE.DAY):
query_id = {
'code': data.iloc[0].code,
'date_stamp': {
'$in': data['date_stamp'].tolist()
}
}
else:
query_id = {
'code': data.iloc[0].code,
'type': freq,
'time_stamp': {
'$in': data['time_stamp'].tolist()
}
}
refcount = coll.count_documents(query_id)
try:
if refcount > 0:
if (len(data) > 1):
# 删掉重复数据
coll.delete_many(query_id)
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
else:
# 持续接收行情，更新记录
if ('created_at' in data.columns):
data.drop('created_at', axis=1, inplace=True)
data = QA_util_to_json_from_pandas(data)
coll.replace_one(query_id, data[0])
else:
# 新 tick，插入记录
data = QA_util_to_json_from_pandas(data)
coll.insert_many(data)
except Exception as e:
if (data is not None):
traceback.print_exception(type(e), e, sys.exc_info()[2])
print(u'save_stock_block_kline failed!!\n', e)
