def QA_fetch_index_day(code,
start,
end,
format='numpy',
collections=DATABASE.index_day):
'获取指数日线'
start = str(start)[0:10]
end = str(end)[0:10]
code = QA_util_code_tolist(code)
if QA_util_date_valid(end) == True:
__data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
batch_size=10000)
if format in ['dict', 'json']:
return [data for data in cursor]
for item in cursor:
__data.append([
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
int(item['up_count']),
int(item['down_count']),
float(item['vol']),
float(item['amount']), item['date']
])
# 多种数据格式
if format in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format in ['list', 'l', 'L']:
__data = __data
elif format in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data,
columns=[
'code', 'open', 'high', 'low', 'close',
'up_count', 'down_count', 'volume',
'amount', 'date'
])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
else:
print(
"QA Error QA_fetch_index_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" "
% format)
return __data
else:
QA_util_log_info('QA something wrong with date')
def QA_fetch_stock_day(code, __start, __end, type_='numpy', collections=QA_Setting.client.quantaxis.stock_day, drop_factor=True):
'获取股票日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
if QA_util_date_valid(__end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6], "date_stamp": {
"$lte": QA_util_date_stamp(__end),
"$gte": QA_util_date_stamp(__start)}}):
if drop_factor:
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date']])
else:
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date'], item['qfqfactor']])
# 多种数据格式
if type_ in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif type_ in ['list', 'l', 'L']:
__data = __data
elif type_ in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'] if drop_factor else [
'code', 'open', 'high', 'low', 'close', 'volume', 'date', 'qfqfactor'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date',drop=False)
return __data
else:
QA_util_log_info('something wrong with date')
def QA_fetch_index_day(code, start, end, format='numpy', collections=DATABASE.index_day):
'获取指数日线'
start = str(start)[0:10]
end = str(end)[0:10]
if QA_util_date_valid(end) == True:
__data = []
cursor = collections.find({
'code': str(code)[0:6], "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}})
if format in ['dict', 'json']:
return [data for data in cursor]
for item in cursor:
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['vol']), item['date']])
# 多种数据格式
if format in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format in ['list', 'l', 'L']:
__data = __data
elif format in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
return __data
else:
QA_util_log_info('something wrong with date')
async def QA_fetch_stock_day(code,
start,
end,
format='numpy',
frequence='day',
collections=DATABASE_ASYNC.stock_day):
'获取股票日线'
start = str(start)[0:10]
end = str(end)[0:10]
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find({
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
})
try:
res = pd.DataFrame([item async for item in cursor])
except SyntaxError:
print('THIS PYTHON VERSION NOT SUPPORT "async for" function')
pass
try:
res = res.drop(
'_id', axis=1).assign(volume=res.vol).query('volume>1').assign(
date=pd.to_datetime(res.date, utc=False)).drop_duplicates(
(['date', 'code'])).set_index('date', drop=False)
res = res.ix[:, [
'code', 'open', 'high', 'low', 'close', 'volume', 'amount',
'date'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_day format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_day data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_stock_full(date_,type_='numpy',collections=QA_Setting.client.quantaxis.stock_day):
#startDate = str(startDate)[0:10]
Date = str(date_)[0:10]
if QA_util_date_valid(Date) == True:
list_a = []
for item in collections.find({
"date_stamp": {
"$lte": QA_util_date_stamp(Date),
"$gte": QA_util_date_stamp(Date)}}):
list_a.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date']])
# 多种数据格式
if type_ in ['n','N','numpy']:
data = numpy.asarray(list_a)
elif type_ in ['list','l','L']:
data = list_a
elif type_ in ['P','p','pandas']:
data_ = numpy.asarray(list_a)
data = DataFrame(list_a, columns=[
'code', 'open', 'high', 'low', 'close', 'volume','date'])
data['date'] = pd.to_datetime(data['date'])
data = data.set_index('date')
data=data[['code', 'open', 'high', 'low', 'close', 'volume']]
return data
else:
QA_util_log_info('something wrong with date')
def _QA_fetch_stock_adj(code,
start,
end,
format='pd',
collections=DATABASE.stock_adj):
"""获取股票复权系数 ADJ
"""
start = str(start)[0:10]
end = str(end)[0:10]
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find(
{
'code': {
'$in': code
},
"date": {
"$lte": end,
"$gte": start
}
}, {"_id": 0},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
res.date = pd.to_datetime(res.date)
return res.set_index('date', drop=False)
def QA_fetch_stock_day(code, startDate, endDate, type_='numpy', collections=QA_Setting.client.quantaxis.stock_day,):
# print(datetime.datetime.now())
startDate = str(startDate)[0:10]
endDate = str(endDate)[0:10]
if QA_util_date_valid(endDate) == True:
list_a = []
for item in collections.find({
'code': str(code)[0:6], "date_stamp": {
"$lte": QA_util_date_stamp(endDate),
"$gte": QA_util_date_stamp(startDate)}}):
list_a.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date']])
# 多种数据格式
if type_ == 'numpy':
data = numpy.asarray(list_a)
elif type_ == 'list':
data = list_a
elif type_ == 'pandas':
data = DataFrame(list_a, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
data['date'] = pd.to_datetime(data['date'])
data = data.set_index('date')
return data
else:
QA_util_log_info('something wrong with date')
def QA_fetch_index_day(code, start, end, format='numpy', collections=DATABASE.index_day):
'获取指数日线'
start = str(start)[0:10]
end = str(end)[0:10]
code = QA_util_code_tolist(code)
if QA_util_date_valid(end) == True:
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}}, {"_id": 0}, batch_size=10000)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(volume=res.vol, date=pd.to_datetime(
res.date)).drop_duplicates((['date', 'code'])).set_index('date', drop=False)
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_index_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_index_day data parameter start=%s end=%s is not right' % (start, end))
def QA_fetch_get_stock_day(name, startDate, endDate, if_fq='01'):
try:
from WindPy import w
except:
QA_util_log_info('No WindPY Module!')
w.start()
if(QA_util_date_valid(endDate) == False):
QA_util_log_info("wrong date")
else:
if if_fq in ['00', 'bfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate)
elif if_fq in ['01', 'qfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=F")
elif if_fq in ['02', 'hfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=B")
else:
QA_util_log_info('wrong fq factor! using qfq')
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=B")
if (data.ErrorCode == 0):
QA_util_log_info("Connent to Wind successfully")
return pd.DataFrame(np.asarray(data.Data).T, columns=data.Fields, index=data.Times)
def QA_fetch_stock_full(date, format='numpy', collections=DATABASE.stock_day):
'获取全市场的某一日的数据'
Date = str(date)[0:10]
if QA_util_date_valid(Date) is True:
__data = []
for item in collections.find({
"date_stamp": {
"$lte": QA_util_date_stamp(Date),
"$gte": QA_util_date_stamp(Date)}}):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['vol']), item['date']])
# 多种数据格式
if format in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format in ['list', 'l', 'L']:
__data = __data
elif format in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
return __data
else:
QA_util_log_info('something wrong with date')
def QA_fetch_stock_full(date, format='numpy', collections=DATABASE.stock_day):
'获取全市场的某一日的数据'
Date = str(date)[0:10]
if QA_util_date_valid(Date) is True:
__data = []
for item in collections.find({
"date_stamp": QA_util_date_stamp(Date)}, batch_size=10000):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['vol']), item['date']])
# 多种数据格式
if format in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format in ['list', 'l', 'L']:
__data = __data
elif format in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
else:
print("QA Error QA_fetch_stock_full format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return __data
else:
QA_util_log_info(
'QA Error QA_fetch_stock_full data parameter date=%s not right' % date)
def QA_fetch_tech_indicator_normalized(code, start, end, keys='all', format='pd', collections=DATABASE.df_tech_indicator_normalized):
"""
keys: 'all' or ['indicator1', indicator2', ....]
return: from start to end, doesn't exclude vol==0 dates in between, those date are continuous
and aligned with sh index already
"""
start = str(start)[0:10]
end = str(end)[0:10]
if keys == 'all':
keys = [ 'OCF_G_q', 'OCF_cal_G_q', 'OCF_netprofit_r_q', 'PB', 'PE', 'PEG', 'PEG_cut',
'PE_cut', 'PNCF', 'POCF', 'PS', 'ROA_q', 'ROE_G_q', 'ROE_q', 'RSI_20',
'STD_12m', 'STD_1m', 'STD_3m', 'STD_6m', 'assetturnover_q',
'bias_turn_1m', 'bias_turn_3m', 'bias_turn_6m', 'captital_tot',
'debitequityratio_q', 'exp_wgt_return_12m', 'exp_wgt_return_1m', 'exp_wgt_return_3m',
'exp_wgt_return_6m', 'fin_leverage_q', 'grossprofit_q', 'profit_G_q', 'profit_cut_G_q',
'profitmargin_q', 'return_12m', 'return_1m', 'return_3m', 'return_6m',
'rs', 'sales_G_q', 'turn_12m', 'turn_1m', 'turn_24m', 'turn_3m',
'turn_6m', 'turnover_d', 'wgt_return_12m', 'wgt_return_1m',
'wgt_return_3m', 'wgt_return_6m']
# code checking
#code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find({
'key': {'$in': keys}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}},
{ str(code): 1, 'date': 1, 'key': 1})
res = pd.DataFrame([item for item in cursor])
try:
# drop '_id', change 'date' from str to datetime, set 'date' to index
res = res.drop('_id', axis=1).assign(date=pd.to_datetime(
res.date)).set_index('date', drop=False)
cols = ['key', str(code)]
res = res.ix[:, cols]
# pivot on 'key' to make each column a indicator
res = res.pivot(columns='key', values=code)
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_tech_indicator format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_tech_indicator data parameter start=%s end=%s is not right' % (start, end))
def QA_fetch_stock_technical_index(code,
start,
end=None,
type='day',
format='pd'):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
if type == 'day':
collections = DATABASE.stock_technical_index
elif type == 'week':
collections = DATABASE.stock_technical_week
elif type == 'month':
collections = DATABASE.stock_technical_month
else:
print("type should be in ['day', 'week', 'month']")
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}, {"_id": 0},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop_duplicates((['code', 'date']))
res['date'] = res['date'].apply(lambda x: str(x)[0:10])
res = res.drop(['date_stamp'], axis=1).set_index(['date', 'code'])
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_technical_index format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_technical_index data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_stock_divyield(code,
start,
end=None,
format='pd',
collections=DATABASE.stock_divyield):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
'a_stockcode': {
'$in': code
},
"dir_dcl_date": {
"$lte": end,
"$gte": start
}
},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop('_id', axis=1).drop_duplicates(
(['dir_dcl_date', 'a_stockcode']))
res = res.ix[:, [
'a_stockcode', 'a_stocksname', 'div_info', 'div_type_code',
'bonus_shr', 'cash_bt', 'cap_shr', 'epsp', 'ps_cr', 'ps_up',
'reg_date', 'dir_dcl_date', 'a_stockcode1', 'ex_divi_date',
'prg'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_divyield format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_divyield data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_get_stock_list(date):
with w.start():
if (QA_util_date_valid(date) == False):
QA_util_log_info("wrong date")
else:
awgs = 'date=' + date + ';sectorid=a001010100000000'
data = w.wset("sectorconstituent", awgs)
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def QA_fetch_stock_financial_calendar(code,
start,
end=None,
format='pd',
collections=DATABASE.report_calendar):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"real_date": {
"$lte": end,
"$gte": start
}
},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop('_id', axis=1).drop_duplicates(
(['report_date', 'code']))
res = res.ix[:, [
'code', 'name', 'pre_date', 'first_date', 'second_date',
'third_date', 'real_date', 'codes', 'report_date', 'crawl_date'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_financial_calendar format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_financial_calendar data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_get_stock_day_simple(name, startDate, endDate):
with w.start():
if (QA_util_date_valid(endDate) == False):
QA_util_log_info("wrong date")
else:
data = w.wsd(name, "sec_name,preclose,open,high,low,close,volume",
startDate, endDate, "Fill=Previous;PriceAdj=F")
#data=w.wsd("000002.SZ", "open,high,low,close,volume", "2017-03-03", "2017-04-01", "PriceAdj=B")
QA_util_log_info(data.ErrorCode)
if (data.ErrorCode == 0):
QA_util_log_info("Connent to Wind successfully")
return data.Data
def QA_fetch_stock_day(code,
start,
end,
format='numpy',
frequence='day',
collections=DATABASE.stock_day):
'获取股票日线'
start = str(start)[0:10]
end = str(end)[0:10]
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end) == True:
__data = []
cursor = collections.find({
'code': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
})
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop(
'_id', axis=1).assign(volume=res.vol).query('volume>1').assign(
date=pd.to_datetime(res.date)).drop_duplicates(
(['date', 'code'])).set_index('date', drop=False)
res = res.ix[:, [
'code', 'open', 'high', 'low', 'close', 'volume', 'amount',
'date'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
return None
else:
QA_util_log_info('something wrong with date')
def QA_fetch_get_stock_list(date):
try:
from WindPy import w
except:
QA_util_log_info('No WindPY Module!')
w.start()
if(QA_util_date_valid(date) == False):
QA_util_log_info("wrong date")
else:
awgs = 'date=' + date + ';sectorid=a001010100000000'
data = w.wset("sectorconstituent", awgs)
return pd.DataFrame(np.asarray(data.Data).T, columns=data.Fields, index=data.Times)
def filter_dates(start, end):
'''
filter dates method
:param start, end: can be str/int of 2019, 201901, '2019-01', '2019-01-01'
:return: string eg： '2019-01-01'
'''
begining_date = '1990-01-01'
start_day = '-01'
start_month = '-01-01'
end_day = '-31'
end_month = '-12-31'
if start == 'all':
start = begining_date
end = str(datetime.date.today())
return start, end
start = str(start)
end = start if end is None else str(end)
if len(start) == 4:
start += start_month
elif len(start) == 6:
start = start[:4] + '-' + start[-2:] + start_day
elif len(start) == 7:
start += start_day
if not QA_util_date_valid(start):
start = begining_date
if len(end) == 4:
end += end_month
elif len(end) == 6:
end = end[:4] + '-' + end[-2:] + end_day
elif len(end) == 7:
end += end_day
if QA_util_date_valid(end):
return start, end
else:
return None, None
def QA_fetch_stock_pure_tech_indicator(code, start, end, vol='non-zero', keys='all', format='pd', collections=DATABASE.stock_tech_indicator_3):
"""
return: from start to end, doesn't exclude vol==0 dates in between, those date are continuous
and aligned with sh index already
"""
start = str(start)[0:10]
end = str(end)[0:10]
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
if vol == 'non-zero':
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)},
'volume': {"$gt": 0}})
else:
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}})
res = pd.DataFrame([item for item in cursor])
try:
# drop '_id', change 'date' from str to datetime, set 'date' to index
res = res.drop('_id', axis=1).assign(date=pd.to_datetime(
res.date)).drop_duplicates((['date', 'code'])).set_index('date', drop=False)
if keys != 'all':
if isinstance(keys, str):
res = res.ix[:, [keys]]
elif isinstance(keys, list):
res = res.ix[:, keys]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_tech_indicator format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_tech_indicator data parameter start=%s end=%s is not right' % (start, end))
def QA_fetch_stock_fianacial(code,
start,
end=None,
format='pd',
collections=DATABASE.stock_financial_analysis):
'获取quant基础数据'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find(
{
'CODE': {
'$in': code
},
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res.columns = [
i.lower() if i == 'CODE' else i for i in list(res.columns)
]
res = res.drop(['date_stamp', '_id'], axis=1).drop_duplicates(
(['code', 'date']))
res['RNG_RES'] = res['AVG60_RNG'] * 60 / res['RNG_60']
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
#res['report_date'] = pd.to_datetime(res['report_date']/1000, unit='s')
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_financial_TTM format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_financial_TTM data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_get_stock_xueqiu(name, startDate, endDate):
with w.start():
if (QA_util_date_valid(endDate) == False):
QA_util_log_info("wrong date")
else:
data = w.wsd(
name,
"xq_accmfocus,xq_accmcomments,xq_accmshares,xq_focusadded,xq_commentsadded,xq_sharesadded,xq_WOW_focus,xq_WOW_comments,xq_WOW_shares",
startDate, endDate, "")
if (data.ErrorCode == 0):
QA_util_log_info("Connent to Wind successfully")
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def QA_fetch_stock_day_adv(code,
start='all',
end=None,
if_drop_index=False,
collections=DATABASE.stock_day):
'获取股票日线'
end = start if end is None else end
start = str(start)[0:10]
end = str(end)[0:10]
if start == 'all':
start = '1990-01-01'
end = str(datetime.date.today())
if isinstance(code, str):
if QA_util_date_valid(end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6],
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}):
__data.append([
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item['vol']),
float(item['amount']), item['date']
])
__data = DataFrame(__data,
columns=[
'code', 'open', 'high', 'low', 'close',
'volume', 'amount', 'date'
])
__data['date'] = pd.to_datetime(__data['date'])
return QA_DataStruct_Stock_day(
__data.query('volume>1').set_index(
['date', 'code'], drop=if_drop_index).sort_index())
else:
QA_util_log_info('something wrong with date')
elif isinstance(code, list):
return QA_DataStruct_Stock_day(
pd.concat(QA_fetch_stocklist_day(
code,
[start, end])).query('volume>1').set_index(['date', 'code'],
drop=if_drop_index))
def QA_fetch_financial_TTM(code,
start,
end=None,
format='pd',
collections=DATABASE.financial_TTM):
'获取财报TTM数据'
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
'CODE': {
'$in': code
},
"date": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop('_id', axis=1).drop_duplicates(
(['REPORT_DATE', 'CODE']))
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_financial_TTM format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_financial_TTM data parameter start=%s end=%s is not right'
% (start, end))
def QA_fetch_get_stock_list_special(date, id):
try:
from WindPy import w
except:
QA_util_log_info('No WindPY Module!')
w.start()
if(QA_util_date_valid(date) == False):
QA_util_log_info("wrong date")
else:
if id in ['big', 'small', 'cixin', 'yujing', 'rzrq', 'rq', 'yj', 'st', 'sst']:
awgs = 'date=' + date + ';sectorid=' + \
data_list.wind_stock_list_special_id[id]
data = w.wset("sectorconstituent", awgs)
return pd.DataFrame(np.asarray(data.Data).T, columns=data.Fields, index=data.Times)
def QA_fetch_stock_day(code, start, end, format='numpy', frequence='day', collections=DATABASE.stock_day):
"""'获取股票日线'
Returns:
[type] -- [description]
感谢@几何大佬的提示
https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-the-specified-fields-and-the-id-field-only
"""
start = str(start)[0:10]
end = str(end)[0:10]
#code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find({
'code': {'$in': code}, "date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)}}, {"_id": 0}, batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(volume=res.vol, date=pd.to_datetime(
res.date)).drop_duplicates((['date', 'code'])).query('volume>1').set_index('date', drop=False)
res = res.ix[:, ['code', 'open', 'high', 'low',
'close', 'volume', 'amount', 'date']]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print("QA Error QA_fetch_stock_day format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format)
return None
else:
QA_util_log_info(
'QA Error QA_fetch_stock_day data parameter start=%s end=%s is not right' % (start, end))
def QA_fetch_get_stock_shape(name, startDate, endDate):
try:
from WindPy import w
except:
QA_util_log_info('No WindPY Module!')
w.start()
if(QA_util_date_valid(endDate) == False):
QA_util_log_info("wrong date")
else:
# history_low近期创历史新低,stage_high近期创阶段新高,history_high近期创历史新高,stage_low近期创阶段新高,up_days连涨天数,down_days连跌天数,breakout_ma向上有效突破均线,breakdown_ma向下有效突破均线,bull_bear_ma均线多空排列看涨看跌
data = w.wsd(name, "history_low,stage_high,history_high,stage_low,up_days,down_days,breakout_ma,breakdown_ma,bull_bear_ma",
startDate, endDate, "n=3;m=60;meanLine=60;N1=5;N2=10;N3=20;N4=30;upOrLow=1")
if (data.ErrorCode == 0):
QA_util_log_info("Connect to Wind successfully")
return pd.DataFrame(np.asarray(data.Data).T, columns=data.Fields, index=data.Times)
def QA_fetch_get_stock_risk(name, startDate, endDate):
with w.start():
if (QA_util_date_valid(endDate) == False):
QA_util_log_info("wrong date")
else:
data = w.wsd(
name,
"annualyeild_100w,annualyeild_24m,annualyeild_60m,annualstdevr_100w,annualstdevr_24m,annualstdevr_60m,beta_100w,beta_24m,beta_60m,avgreturn,avgreturny,stdevry,stdcof,risk_nonsysrisk1,r2,alpha2,beta,sharpe,treynor,jensen,jenseny,betadf",
startDate, endDate,
"period=2;returnType=1;index=000001.SH;yield=1")
if (data.ErrorCode == 0):
QA_util_log_info("Connent to Wind successfully")
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def QA_fetch_interest_rate(start,
end=None,
format='pd',
collections=DATABASE.interest_rate):
'获取股票日线'
#code= [code] if isinstance(code,str) else code
# code checking
if end is None:
end = QA_util_today_str()
if start is None:
start = '1999-01-01'
if QA_util_date_valid(end):
__data = []
cursor = collections.find(
{
"date_stamp": {
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
}, {"_id": 0},
batch_size=10000)
#res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop(columns=['crawl_date', 'date_stamp'])
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error Interest Rate format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format)
return None
else:
QA_util_log_info(
'QA Error Interest Rate data parameter start=%s end=%s is not right'
% (start, end))