def QA_SU_save_deal(dealist, client=DATABASE):
"""存储order_handler的deal_status
Arguments:
dealist {[dataframe]} -- [description]
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
if isinstance(dealist, pd.DataFrame):
collection = client.deal
collection.create_index(
[('account_cookie',
ASCENDING),
('trade_id',
ASCENDING)],
unique=True
)
try:
dealist = QA_util_to_json_from_pandas(dealist.reset_index())
collection.insert_many(dealist, ordered=False)
except Exception as e:
pass
def __init__(self, market_data):
"""转化成dict模式
Arguments:
market_data {[type]} -- [description]
"""
if isinstance(market_data, dict):
self.market_data = market_data
elif isinstance(market_data, pd.DataFrame):
self.market_data = QA_util_to_json_from_pandas(market_data)
def __saving_work(code):
QA_util_log_info('##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format(
str(code)),
ui_log=ui_log)
try:
coll.insert_many(
QA_util_to_json_from_pandas(
# 🛠todo str(stock_list[code]) 参数不对？
QA_fetch_get_stock_transaction(str(code),
str(stock_list[code]),
str(now_time())[0:10])))
except:
err.append(str(code))
def __saving_work(code, coll_stock_year):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_YEAR==== {}'.format(
str(code)))
ref = coll_stock_year.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
coll_stock_year.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code),
QA_util_get_next_day(start_date),
end_date,
'00',
frequence='year')))
else:
start_date = '1990-01-01'
QA_util_log_info(
'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
coll_stock_year.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code),
start_date,
end_date,
'00',
frequence='year')))
except:
err.append(str(code))
def __saving_work(code, stock_shares):
try:
QA_util_log_info(
'##JOB01 Now Saving SSINA shares change==== {}'.format(
str(code)), ui_log)
stock_shares.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_shares_sina(code)),
ordered=False)
gc.collect()
except Exception as error0:
print(error0)
err.append(str(code))
def __saving_work(code, stock_financial_sina):
try:
QA_util_log_info(
'##JOB01 Now Saving SINA financial_report==== {}'.format(
str(code)), ui_log)
stock_financial_sina.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_report_sina(code, YEARS)),
ordered=False)
gc.collect()
except Exception as error0:
print(error0)
err.append(str(code))
def QA_SU_save_deal(dealist, client=DATABASE):
"""存储order_handler的deal_status
Arguments:
dealist {[dataframe]} -- [description]
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
if isinstance(dealist, pd.DataFrame):
dealist = QA_util_to_json_from_pandas(dealist.reset_index())
collection = client.deal
collection.create_index([('account_cookie', ASCENDING),
('trade_id', ASCENDING)],
unique=True)
try:
collection.insert_many(dealist, ordered=False)
except Exception as e:
pass
def QA_fetch_stock_fianacial(code,
start,
end=None,
format='pd',
collections=DATABASE.stock_financial_analysis):
'获取quant基础数据'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find(
{
'CODE': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res.columns = [
i.lower() if i == 'CODE' else i for i in list(res.columns)
]
res = res.drop(['date_stamp', '_id'], axis=1).drop_duplicates(
(['code', 'date']))
res['RNG_RES'] = res['AVG60_RNG'] * 60 / res['RNG_60']
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
#res['report_date'] = pd.to_datetime(res['report_date']/1000, unit='s')
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_financial_TTM format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_financial_TTM data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_index_day(code,
start,
end,
format='numpy',
collections=DATABASE.index_day):
'获取指数日线'
start = str(start)[0:10]
end = str(end)[0:10]
code = QA_util_code_tolist(code)
if QA_util_date_valid(end) == True:
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}, {"_id": 0},
batch_size=10000)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(volume=res.vol,
date=pd.to_datetime(res.date)).drop_duplicates(
(['date', 'code'
])).query('volume>1').set_index('date',
drop=False)
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_index_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_index_day data parameter start=%s end=%s is not right'
% (start, end))
def QA_ts_update_namechange():
"""
保存所有股票的历史曾用名
"""
# 建表
coll = DATABASE.namechange
coll.create_index(
[
("code", ASCENDING),
("start_date_stamp", ASCENDING),
("end_date_stamp", ASCENDING),
("ann_date_stamp", ASCENDING),
],
unique=True,
)
# 初始化数据接口
pro = get_pro()
# 获取历史所有股票
symbol_list = sorted(
list(set(QA_fmt_code_list(QA_fetch_stock_basic().index.tolist(),
"ts"))))
df = pd.DataFrame()
for i, symbol in enumerate(symbol_list):
if i % 100 == 0:
print(f"Saving {i}th stock name, stock is {symbol}")
try:
df = df.append(pro.namechange(ts_code=symbol))
except:
time.sleep(1)
try:
df = df.append(pro.namechange(ts_code=symbol))
except:
raise ValueError("[ERROR]\t数据获取失败")
# df.to_csv("test.csv")
df["code"] = QA_fmt_code_list(df["ts_code"])
df["start_date_stamp"] = df["start_date"].apply(QA_util_date_stamp)
df["end_date_stamp"] = df["end_date"].apply(QA_util_date_stamp)
df["ann_date_stamp"] = df["ann_date"].apply(QA_util_date_stamp)
df = df.where(pd.notnull(df), None)
js = QA_util_to_json_from_pandas(df.drop(columns=["ts_code"]))
for item in js:
if not item["end_date"]:
item["end_date_stamp"] = 9999999999
qry = {
"code": item["code"],
"start_date_stamp": item["start_date_stamp"],
"end_date_stamp": item["end_date_stamp"],
"ann_date_stamp": item["ann_date_stamp"],
}
if coll.count_documents(qry) == 0:
coll.insert_one(item)
def __saving_work(code, coll):
QA_util_log_info('##JOB07 Now Saving ETF_MIN ==== {}'.format(
str(code)))
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find({'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
QA_util_log_info(
'##JOB07.{} Now Saving {} from {} to {} =={} '.format(
['1min',
'5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
QA_util_log_info(
'##JOB07.{} Now Saving {} from {} to {} =={} '.format(
['1min',
'5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data))
except:
err.append(code)
def QA_fetch_stock_alpha(code,
start,
end=None,
format='pd',
collections=DATABASE.stock_alpha):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}, {"_id": 0},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop_duplicates((['code',
'date'])).set_index(['date', 'code'])
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_alpha format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_alpha data parameter start=%s end=%s is not right'
% (start, end))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_hfq = QA_fetch_get_stock_day(
i,
start='1990-01-01',
if_fq='02',
type_='pd'
)
data_json = QA_util_to_json_from_pandas(data_hfq)
__coll.insert_many(data_json)
except Exception as e:
print(e)
QA_util_log_info('error in saving ==== %s' % str(i))
def get(self):
action = self.get_argument('action', 'acchistory')
if action == 'accountlist':
res = self.manager.get_allaccountname()
self.write({'res': res})
elif action == 'portfoliolist':
res = self.manager.get_allportfolio()
self.write({'res': res})
elif action == 'accountinportfolio':
portfolio = self.get_argument('portfolio', 't12')
res = self.manager.get_portfolio_panel(portfolio)
self.write({'res': QA_util_to_json_from_pandas(res)})
def QA_fetch_get_trade_date(end, exchange):
data = QATs.trade_cal()
da = data[data.isOpen > 0]
data_json = QA_util_to_json_from_pandas(data)
message = []
for i in range(0, len(data_json) - 1, 1):
date = data_json[i]['calendarDate']
num = i + 1
exchangeName = 'SSE'
data_stamp = QA_util_date_stamp(date)
mes = {'date': date, 'num': num,
'exchangeName': exchangeName, 'date_stamp': data_stamp}
message.append(mes)
return message
def __saving_work(code, coll):
QA_util_log_info(
'##JOB02 Now Saving XDXR INFO ==== {}'.format(str(code)),
ui_log=ui_log
)
try:
coll.insert_many(
QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr(str(code))),
ordered=False
)
except:
err.append(str(code))
def _ts_update_all(report_date, report_type, sheet_type, wait_seconds,
max_trial):
# 获取指定报告期，指定报告类型，指定报表类型的截面数据
df = QA_fetch_get_crosssection_financial(
report_date=report_date,
report_type=report_type,
sheet_type=sheet_type,
wait_seconds=wait_seconds,
max_trial=max_trial,
)
coll = eval(f"DATABASE.{sheet_type}")
# 考虑到查找的方式，可能根据股票代码查找，可能根据报告期查找，可能根据公告期查找，可能根据最后公告期查找，可能根据报告类型查找
coll.create_index(
[
("code", ASCENDING),
("report_label", ASCENDING),
("report_date", DESCENDING),
("report_type", ASCENDING),
("report_date_stamp", ASCENDING),
("ann_date_stamp", ASCENDING),
("f_ann_date_stamp", ASCENDING),
],
unique=False,
)
# FIXME: insert_many may be better
for item in QA_util_to_json_from_pandas(df):
report_label = None
if item["report_date"].endswith("0331"):
report_label = "1"
elif item["report_date"].endswith("0630"):
report_label = "2"
elif item["report_date"].endswith("0930"):
report_label = "3"
elif item["report_date"].endswith("1231"):
report_label = "4"
item["report_label"] = report_label
coll.update_one(
{
"code": item["code"],
"report_label": report_label,
"report_date": item["report_date"],
"report_type": item["report_type"],
"report_date_stamp": QA_util_date_stamp(
item["report_date"]),
"ann_date_stamp": QA_util_date_stamp(item["ann_date"]),
"f_ann_date_stamp": QA_util_date_stamp(item["f_ann_date"]),
},
{"$set": item},
upsert=True,
)
def QA_SU_save_order(orderlist, client=DATABASE):
"""存储order_handler的order_status
Arguments:
orderlist {[dataframe]} -- [description]
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
orderlist = QA_util_to_json_from_pandas(orderlist.reset_index())
collection = client.order
collection.create_index(
[('account_cookie', ASCENDING), ('realorder_id', ASCENDING)], unique=True)
try:
for item in orderlist:
if item:
#item['date']= QA_util_get_order_day()
collection.update_one({'account_cookie': item.get('account_cookie'), 'realorder_id': item.get('realorder_id')},
{'$set': item}, upsert=True)
except Exception as e:
print(e)
pass
def QA_SU_save_order(orderlist, client=DATABASE):
"""存储order_handler的order_status
Arguments:
orderlist {[dataframe]} -- [description]
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
if isinstance(orderlist, pd.DataFrame):
collection = client.order
collection.create_index(
[('account_cookie',
ASCENDING),
('realorder_id',
ASCENDING)],
unique=True
)
try:
orderlist = QA_util_to_json_from_pandas(orderlist.reset_index())
for item in orderlist:
if item:
#item['date']= QA_util_get_order_day()
collection.update_one(
{
'account_cookie': item.get('account_cookie'),
'realorder_id': item.get('realorder_id')
},
{'$set': item},
upsert=True
)
except Exception as e:
print(e)
pass
async def QA_fetch_stock_min(code, start, end, format='numpy', frequence='1min', collections=DATABASE_ASYNC.stock_min):
'获取股票分钟线'
if frequence in ['1min', '1m']:
frequence = '1min'
elif frequence in ['5min', '5m']:
frequence = '5min'
elif frequence in ['15min', '15m']:
frequence = '15min'
elif frequence in ['30min', '30m']:
frequence = '30min'
elif frequence in ['60min', '60m']:
frequence = '60min'
else:
print("QA Error QA_fetch_stock_min parameter frequence=%s is none of 1min 1m 5min 5m 15min 15m 30min 30m 60min 60m" % frequence)
__data = []
# code checking
code = QA_util_code_tolist(code)
cursor = collections.find({
'code': {'$in': code}, "time_stamp": {
"$gte": QA_util_time_stamp(start),
"$lte": QA_util_time_stamp(end)
}, 'type': frequence
})
try:
res = pd.DataFrame([item async for item in cursor])
except SyntaxError:
print('THIS PYTHON VERSION NOT SUPPORT "async for" function')
pass
try:
res = res.drop('_id', axis=1).assign(volume=res.vol).query('volume>1').assign(datetime=pd.to_datetime(
res.datetime)).drop_duplicates(['datetime', 'code']).set_index('datetime', drop=False)
# return res
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_stock_min format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
def QA_fetch_stock_day(code, start, end, format='numpy', frequence='day', collections=DATABASE.stock_day):
"""'获取股票日线'
Returns:
[type] -- [description]
感谢@几何大佬的提示
https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-the-specified-fields-and-the-id-field-only
"""
start = str(start)[0:10]
end = str(end)[0:10]
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}}, {"_id": 0}, batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(volume=res.vol, date=pd.to_datetime(
res.date)).drop_duplicates((['date', 'code'])).query('volume>1').set_index('date', drop=False)
res = res.ix[:, ['code', 'open', 'high', 'low',
'close', 'volume', 'amount', 'date']]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_stock_day format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_day data parameter start=%s end=%s is not right' % (start, end))
def __saving_work(code, coll):
QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code)))
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find({'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
QA_util_log_info(
'##JOB03.%s Now Saving %s from %s to %s ==%s ' % ([
'1min', '5min', '15min', '30min', '60min'
].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
QA_util_log_info(
'##JOB03.%s Now Saving %s from %s to %s ==%s ' % ([
'1min', '5min', '15min', '30min', '60min'
].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data))
except Exception as e:
QA_util_log_info(e)
err.append(code)
def QA_fetch_interest_rate(start,
end=None,
format='pd',
collections=DATABASE.interest_rate):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
if end is None:
end = QA_util_today_str()
if start is None:
start = '1999-01-01'
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}, {"_id": 0},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop(columns=['crawl_date', 'date_stamp'])
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error Interest Rate format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error Interest Rate data parameter start=%s end=%s is not right'
% (start, end))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_bfq = QATushare.QA_fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='00',
type_='pd')
data_qfq = QATushare.QA_fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='01',
type_='pd')
data_qfq['qfqfactor'] = data_qfq['open'] / data_bfq['open']
data_json = QA_util_to_json_from_pandas(data_qfq)
__coll.insert_many(data_json)
except:
QA_util_log_info('error in saving ==== %s' % str(i))
def QA_SU_save_daily_basic(client=DATABASE, date=None):
__coll = client.daily_basic_tushare
'''
__coll.create_index(
[("ts_code",
pymongo.ASCENDING),
("trade_date",
pymongo.ASCENDING)]
)'''
if (date == None):
# date = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y%m%d')
date = trading_calendar_utils.get_last_x_trading_day_from_mongodb()
date = str(date)
data = QA_fetch_get_stock_daily_basic(date)
if (data is not None):
__coll.insert_many(QA_util_to_json_from_pandas(data))
def fetch_index_day_common(field_name, field_value, start, end,collections, format='pd'):
'通用数据查询'
start = str(start)[0:10]
end = str(end)[0:10]
field_value_ = field_value
if not isinstance(field_value_, list):
field_value_ = [field_value_]
cursor = collections.find(
{
field_name: {
'$in': field_value_
},
"date_stamp":
{
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
{"_id": 0},
batch_size=10000
)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(
date=pd.to_datetime(res.date, utc=False)
).set_index('date',drop=True)
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"Error fetch_index_day_em format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" "
% format
)
return None
def QA_save_fq_factor(client=QA_Setting.client):
'保存复权因子'
_coll = client.quantaxis.fq_factor
_stock_list = QA_fetch_get_stock_list()
for _item in _stock_list:
data = QA_data_fq_factor(_item)
data['date'] = data.index
data['date'] = data['date'].apply(lambda x: str(x)[0:10])
data['date_stamp'] = data['date'].apply(
lambda x: QA_util_date_stamp(x))
data['code'] = _item
data_json = QA_util_to_json_from_pandas(data)
_coll.insert_many(data_json)
def QA_SU_save_stock_block(client=DATABASE, ui_log=None, ui_progress=None):
"""
Tushare的版块数据
Returns:
[type] -- [description]
"""
coll = client.stock_block
coll.create_index('code')
try:
# 暂时先只有中证500
csindex500 = QA_fetch_get_stock_block()
coll.insert_many(QA_util_to_json_from_pandas(csindex500))
QA_util_log_info('SUCCESS save stock block ^_^', ui_log)
except Exception as e:
QA_util_log_info('ERROR CODE: ', ui_log)
QA_util_log_info(e, ui_log)
def getMin(cls, code, start, end, if_fq='00', frequence=8):
collections = cls.collectionsMin
'获取股票分钟线'
_, type_, _ = cls.getReverseFrequence(frequence)
_data = []
# code checking
code = QA_util_code_tolist(code)
cursor = collections.find(
{
'code': {
'$in': code
},
"time_stamp": {
"$gte": QA_util_time_stamp(start),
"$lte": QA_util_time_stamp(end)
},
'type': type_
}, {"_id": 0},
batch_size=10000)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(
volume=res.vol, datetime=pd.to_datetime(
res.datetime)).query('volume>1').drop_duplicates(
['datetime', 'code']).set_index('datetime', drop=False)
# return res
except:
res = None
if cls.format in ['P', 'p', 'pandas', 'pd']:
return res
elif cls.format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif cls.format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif cls.format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_min format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% cls.format)
return None
def QA_fetch_tech_indicator(code, start, end, keys='all', format='pd', collections=DATABASE.df_tech_indicator):
"""
return: from start to end, doesn't exclude vol==0 dates in between, those date are continuous
and aligned with sh index already
"""
start = str(start)[0:10]
end = str(end)[0:10]
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}})
res = pd.DataFrame([item for item in cursor])
try:
# drop '_id', change 'date' from str to datetime, set 'date' to index
res = res.drop('_id', axis=1).assign(date=pd.to_datetime(
res.date)).drop_duplicates((['date', 'code'])).set_index('date', drop=False)
if keys != 'all':
if isinstance(keys, str):
res = res.ix[:, [keys]]
elif isinstance(keys, list):
res = res.ix[:, keys]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_tech_indicator format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_tech_indicator data parameter start=%s end=%s is not right' % (start, end))
def QA_SU_save_stock_fianacial_momgo(start_date=None,end_date=None, ui_log = None, ui_progress = None):
if start_date is None:
if end_date is None:
start_date = QA_util_today_str()
end_date = start_date
elif end_date is not None:
start_date = '2008-01-01'
elif start_date is not None:
if end_date == None:
end_date = QA_util_today_str()
elif end_date is not None:
if end_date < start_date:
print('end_date should large than start_date')
col = DATABASE.stock_financial_analysis
col.create_index(
[("CODE", ASCENDING), ("date_stamp", ASCENDING)], unique=True)
deal_date_list = list(pd.date_range(start_date, end_date).map(lambda t:str(t.date())))
if deal_date_list is None:
print('not a trading day')
else:
for deal_date in deal_date_list:
data = QA_util_etl_stock_quant(deal_date)
if data is not None:
data = data.drop_duplicates(
(['CODE', 'date']))
QA_util_log_info(
'##JOB01 Pre Data stock financial data ============== {deal_date} '.format(deal_date=deal_date), ui_log)
data = QA_util_to_json_from_pandas(data)
print("got stock financial data ============== {deal_date}".format(deal_date=deal_date))
QA_util_log_info(
'##JOB02 Got Data stock financial data ============== {deal_date}'.format(deal_date=deal_date), ui_log)
try:
col.insert_many(data, ordered=False)
QA_util_log_info(
'##JOB03 Now stock financial data saved ============== {deal_date} '.format(deal_date=deal_date), ui_log)
except Exception as e:
if isinstance(e, MemoryError):
col.insert_many(data, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
pass
else:
QA_util_log_info(
'##JOB01 No Data stock_fianacial_data ============== {deal_date} '.format(deal_date=deal_date), ui_log)
def __saving_work(self, df=pd.DataFrame()):
try:
if not (df is None) and len(df) > 0:
coll_stock_day = self.client.stock_day
coll_stock_day.create_index([("code", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)
])
coll_stock_day.insert_many(QA_util_to_json_from_pandas(df))
QA_util_log_info(
'##JOB02 Now Saved STOCK_DAY==== {}'.format(
df.code.unique()[0]), self.ui_log)
else:
QA_util_log_info(
'##JOB02 Saving STOCK_DAY==== {}'.format('skipped'),
self.ui_log)
except Exception as error0:
print(error0, flush=True)
def QA_SU_save_stock_list(client=DATABASE):
"""save stock_list
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
client.drop_collection('stock_list')
coll = client.stock_list
coll.create_index('code')
err = []
try:
QA_util_log_info('##JOB08 Now Saving STOCK_LIST ====')
coll.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_list()))
except:
pass
class _realtime_base():
"""
realtime 基类
主要字段有:
code/name
time
open/high/low
买卖报价队列:(不同的可能不一样 只提供list)
ask_list[ask1_price/ask1_volume|ask2_price/ask2_volume|ask3_price/ask3_volume....]
bid_list[bid1_price/bid1_volume|bid2_price/bid2_volume|bid3_price/bid3_volume....]
"""
def __init__(self, market_data):
"""转化成dict模式
Arguments:
market_data {[type]} -- [description]
"""
if isinstance(market_data, dict):
self.market_data = market_data
elif isinstance(market_data, pd.DataFrame):
self.market_data = QA_util_to_json_from_pandas(market_data)
@property
def open(self):
return self.market_data.get('open', None)
@property
def price(self):
return self.market_data.get('price', None)
@property
def datetime(self):
return self.market_data.get('datetime', None)
@property
def high(self):
return self.market_data.get('high', None)
@property
def low(self):
return self.market_data.get('low', None)
@property
def code(self):
return self.market_data.get('code', None)
@property
def last_close(self):
return self.market_data.get('last_close', None)
@property
def cur_vol(self):
return self.market_data.get('cur_vol', None)
@property
def bid1(self):
return self.market_data.get('bid1', None)
@property
def bid_vol1(self):
return self.market_data.get('bid_vol1', None)
@property
def bid2(self):
return self.market_data.get('bid2', None)
@property
def bid_vol2(self):
return self.market_data.get('bid_vol2', None)
@property
def bid3(self):
return self.market_data.get('bid3', None)
@property
def bid_vol3(self):
return self.market_data.get('bid_vol3', None)
@property
def bid4(self):
return self.market_data.get('bid4', None)
@property
def bid_vol4(self):
return self.market_data.get('bid_vol4', None)
@property
def bid5(self):
return self.market_data.get('bid5', None)
@property
def bid_vol5(self):
return self.market_data.get('bid_vol5', None)
@property
def ask1(self):
return self.market_data.get('ask1', None)
@property
def ask_vol1(self):
return self.market_data.get('ask_vol1', None)
@property
def ask2(self):
return self.market_data.get('ask2', None)
@property
def ask_vol2(self):
return self.market_data.get('ask_vol2', None)
@property
def ask3(self):
return self.market_data.get('ask3', None)
@property
def ask_vol3(self):
return self.market_data.get('ask_vol3', None)
@property
def ask4(self):
return self.market_data.get('ask4', None)
@property
def ask_vol4(self):
return self.market_data.get('ask_vol4', None)
@property
def ask5(self):
return self.market_data.get('ask5', None)
@property
def ask_vol5(self):
return self.market_data.get('ask_vol5', None)
