def local_get_stock_xdxr(code, format='pd', collections=DATABASE.stock_xdxr):
'获取股票除权信息/数据库'
code = QA_util_code_tolist(code)
data = pd.DataFrame(
[
item for item in
collections.find({'code': {
'$in': code
}},
batch_size=10000)
]
).drop(['_id'],
axis=1)
data['date'] = pd.to_datetime(data['date'])
return data.set_index('date', drop=False)
def local_get_stock_info(code, format='pd', collections=DATABASE.stock_info):
code = QA_util_code_tolist(code)
try:
data = pd.DataFrame(
[
item for item in collections
.find({'code': {
'$in': code
}},
{"_id": 0},
batch_size=10000)
]
)
# data['date'] = pd.to_datetime(data['date'])
return data.set_index('code', drop=False)
except Exception as e:
print(e)
return None
def local_get_stock_block(
code=None,
format='pd',
collections=DATABASE.stock_block
):
if code is not None:
code = QA_util_code_tolist(code)
data = pd.DataFrame(
[
item for item in
collections.find({'code': {
'$in': code
}},
batch_size=10000)
]
).drop(['_id'],
axis=1)
return data.set_index('code', drop=False)
else:
data = pd.DataFrame([item for item in collections.find()]
).drop(['_id'],
axis=1)
return data.set_index('code', drop=False)
def local_get_stock_sw_block(code, date, collections=JQDATA.industry):
code = QA_util_code_tolist(code)
date_stamp = time.mktime(time.strptime(date, '%Y-%m-%d'))
try:
data = pd.DataFrame(
[
item for item in collections
.find({'code': {
'$in': code
},
'date_stamp': date_stamp
},
{"_id": 0},
batch_size=10000)
]
)
ind = data.pivot(index='code', columns='type', values='ind_name')
ind['sw'] = ind.apply(lambda x: '-'.join([x['sw1'], x['sw2'], x['sw3']]), axis=1)
# data['date'] = pd.to_datetime(data['date'])
return ind
except Exception as e:
print(e)
return None
def local_get_stock_adj(
code,
start,
end,
format='pd',
collections=DATABASE.stock_adj
):
"""获取股票复权系数 ADJ
"""
start = str(start)[0:10]
end = str(end)[0:10]
# code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find(
{
'code': {
'$in': code
},
"date": {
"$lte": end,
"$gte": start
}
},
{"_id": 0},
batch_size=10000
)
# res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
res.date = pd.to_datetime(res.date)
return res.set_index('date', drop=False)
def local_get_ctp_tick(
code,
start,
end,
frequence,
format='pd',
collections=DATABASE.ctp_tick
):
"""仅供存储的ctp tick使用
Arguments:
code {[type]} -- [description]
Keyword Arguments:
format {str} -- [description] (default: {'pd'})
collections {[type]} -- [description] (default: {DATABASE.ctp_tick})
Returns:
[type] -- [description]
"""
code = QA_util_code_tolist(code, auto_fill=False)
cursor = collections.find(
{
'InstrumentID': {
'$in': code
},
"time_stamp":
{
"$gte": QA_util_time_stamp(start),
"$lte": QA_util_time_stamp(end)
},
'type': frequence
},
{"_id": 0},
batch_size=10000
)
hq = pd.DataFrame([data for data in cursor]
).replace(1.7976931348623157e+308,
numpy.nan).replace('',
numpy.nan).dropna(axis=1)
p1 = hq.loc[:,
[
'ActionDay',
'AskPrice1',
'AskVolume1',
'AveragePrice',
'BidPrice1',
'BidVolume1',
'HighestPrice',
'InstrumentID',
'LastPrice',
'OpenInterest',
'TradingDay',
'UpdateMillisec',
'UpdateTime',
'Volume'
]]
p1 = p1.assign(
datetime=p1.ActionDay.apply(QA_util_date_int2str) + ' ' +
p1.UpdateTime +
(p1.UpdateMillisec / 1000000).apply(lambda x: str('%.6f' % x)[1:]),
code=p1.InstrumentID
)
p1.datetime = pd.to_datetime(p1.datetime)
return p1.set_index(p1.datetime)
def local_get_future_min(
code,
start,
end,
format='numpy',
frequence='1min',
collections=DATABASE.future_min
):
'获取股票分钟线'
if frequence in ['1min', '1m']:
frequence = '1min'
elif frequence in ['5min', '5m']:
frequence = '5min'
elif frequence in ['15min', '15m']:
frequence = '15min'
elif frequence in ['30min', '30m']:
frequence = '30min'
elif frequence in ['60min', '60m']:
frequence = '60min'
_data = []
code = QA_util_code_tolist(code, auto_fill=False)
cursor = collections.find(
{
'code': {
'$in': code
},
"time_stamp":
{
"$gte": QA_util_time_stamp(start),
"$lte": QA_util_time_stamp(end)
},
'type': frequence
},
batch_size=10000
)
if format in ['dict', 'json']:
return [data for data in cursor]
for item in cursor:
_data.append(
[
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item.get('position',
0)),
float(item.get('price',
0)),
float(item.get('trade',
item.get('volume',
0))),
item['datetime'],
item['tradetime'],
item['time_stamp'],
item['date'],
item['type']
]
)
_data = DataFrame(
_data,
columns=[
'code',
'open',
'high',
'low',
'close',
'position',
'price',
'trade',
'datetime',
'tradetime',
'time_stamp',
'date',
'type'
]
)
_data = _data.assign(datetime=pd.to_datetime(_data['datetime'])
).drop_duplicates((['datetime',
'code'])).set_index(
'datetime',
drop=False
)
if format in ['numpy', 'np', 'n']:
return numpy.asarray(_data)
elif format in ['list', 'l', 'L']:
return numpy.asarray(_data).tolist()
elif format in ['P', 'p', 'pandas', 'pd']:
return _data
def local_get_future_day(
code,
start,
end,
format='numpy',
collections=DATABASE.future_day
):
start = str(start)[0:10]
end = str(end)[0:10]
code = QA_util_code_tolist(code, auto_fill=False)
if QA_util_date_valid(end) == True:
_data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp":
{
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
{"_id": 0},
batch_size=10000
)
if format in ['dict', 'json']:
return [data for data in cursor]
for item in cursor:
_data.append(
[
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item['position']),
float(item['price']),
float(item['trade']),
item['date']
]
)
# 多种数据格式
if format in ['n', 'N', 'numpy']:
_data = numpy.asarray(_data)
elif format in ['list', 'l', 'L']:
_data = _data
elif format in ['P', 'p', 'pandas', 'pd']:
_data = DataFrame(
_data,
columns=[
'code',
'open',
'high',
'low',
'close',
'position',
'price',
'trade',
'date'
]
).drop_duplicates()
_data['date'] = pd.to_datetime(_data['date'])
_data = _data.set_index('date', drop=False)
else:
print(
"QA Error QA_fetch_future_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" "
% format
)
return _data
else:
print('QA something wrong with date')
def local_get_index_min(
code,
start,
end,
format='numpy',
frequence='1min',
collections=DATABASE.index_min
):
'获取指数分钟线'
if frequence in ['1min', '1m']:
frequence = '1min'
elif frequence in ['5min', '5m']:
frequence = '5min'
elif frequence in ['15min', '15m']:
frequence = '15min'
elif frequence in ['30min', '30m']:
frequence = '30min'
elif frequence in ['60min', '60m']:
frequence = '60min'
_data = []
code = QA_util_code_tolist(code)
cursor = collections.find(
{
'code': {
'$in': code
},
"time_stamp":
{
"$gte": QA_util_time_stamp(start),
"$lte": QA_util_time_stamp(end)
},
'type': frequence
},
{"_id": 0},
batch_size=10000
)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(
volume=res.vol,
datetime=pd.to_datetime(res.datetime)
).query('volume>1').drop_duplicates(['datetime',
'code']).set_index(
'datetime',
drop=False
)
# return res
except:
res = None
# 多种数据格式
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_index_min format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format
)
return None
def local_get_index_day(
code,
start,
end,
format='numpy',
collections=DATABASE.index_day
):
'获取指数日线'
start = str(start)[0:10]
end = str(end)[0:10]
code = QA_util_code_tolist(code)
if QA_util_date_valid(end) == True:
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp":
{
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
{"_id": 0},
batch_size=10000
)
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(
volume=res.vol,
date=pd.to_datetime(res.date)
).drop_duplicates((['date',
'code'])).set_index(
'date',
drop=False
)
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_index_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" "
% format
)
return None
else:
print(
'QA Error QA_fetch_index_day data parameter start=%s end=%s is not right'
% (start,
end)
)
def local_get_stock_day(
code,
start,
end,
format='numpy',
frequence='day',
collections=DATABASE.stock_day
):
"""'获取股票日线'
Returns:
[type] -- [description]
感谢@几何大佬的提示
https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-the-specified-fields-and-the-id-field-only
"""
start = str(start)[0:10]
end = str(end)[0:10]
# code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
cursor = collections.find(
{
'code': {
'$in': code
},
"date_stamp":
{
"$lte": QA_util_date_stamp(end),
"$gte": QA_util_date_stamp(start)
}
},
{"_id": 0},
batch_size=10000
)
# res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.assign(
volume=res.vol,
date=pd.to_datetime(res.date)
).drop_duplicates((['date',
'code'])).query('volume>1').set_index(
'date',
drop=False
)
res = res.loc[:,
[
'code',
'open',
'high',
'low',
'close',
'volume',
'amount',
'date'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_day format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format
)
return None
else:
print(
'QA Error QA_fetch_stock_day data parameter start=%s end=%s is not right'
% (start,
end)
)
def local_get_stock_divyield(
code,
start,
end=None,
format='pd',
collections=DATABASE.stock_divyield
):
'获取股票日线'
# code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
_data = []
cursor = collections.find(
{
'a_stockcode': {
'$in': code
},
"dir_dcl_date": {
"$lte": end,
"$gte": start
}
},
{"_id": 0},
batch_size=10000
)
# res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop_duplicates((['dir_dcl_date', 'a_stockcode']))
res = res.loc[:,
[
'a_stockcode',
'a_stocksname',
'div_info',
'div_type_code',
'bonus_shr',
'cash_bt',
'cap_shr',
'epsp',
'ps_cr',
'ps_up',
'reg_date',
'dir_dcl_date',
'a_stockcode1',
'ex_divi_date',
'prg'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_divyield format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format
)
return None
else:
print(
'QA Error QA_fetch_stock_divyield data parameter start=%s end=%s is not right'
% (start,
end)
)
def local_get_stock_financial_calendar(
code,
start,
end=None,
format='pd',
collections=DATABASE.report_calendar
):
'获取股票日线'
# code= [code] if isinstance(code,str) else code
# code checking
code = QA_util_code_tolist(code)
if QA_util_date_valid(end):
_data = []
cursor = collections.find(
{
'code': {
'$in': code
},
"real_date": {
"$lte": end,
"$gte": start
}
},
{"_id": 0},
batch_size=10000
)
# res=[QA_util_dict_remove_key(data, '_id') for data in cursor]
res = pd.DataFrame([item for item in cursor])
try:
res = res.drop_duplicates((['report_date', 'code']))
res = res.loc[:,
[
'code',
'name',
'pre_date',
'first_date',
'second_date',
'third_date',
'real_date',
'codes',
'report_date',
'crawl_date'
]]
except:
res = None
if format in ['P', 'p', 'pandas', 'pd']:
return res
elif format in ['json', 'dict']:
return QA_util_to_json_from_pandas(res)
# 多种数据格式
elif format in ['n', 'N', 'numpy']:
return numpy.asarray(res)
elif format in ['list', 'l', 'L']:
return numpy.asarray(res).tolist()
else:
print(
"QA Error QA_fetch_stock_financial_calendar format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" "
% format
)
return None
else:
print(
'QA Error QA_fetch_stock_financial_calendar data parameter start=%s end=%s is not right'
% (start,
end)
)