def fetch_stock_full(date_, format_='numpy', collections=ms.client.flyshare.stock_day):
'获取全市场的某一日的数据'
#__start = str(__start)[0:10]
Date = str(date_)[0:10]
if util_date_valid(Date) == True:
__data = []
for item in collections.find({
"date_stamp": {
"$lte": util_date_stamp(Date),
"$gte": util_date_stamp(Date)}}):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date']])
# 多种数据格式
if format_ in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format_ in ['list', 'l', 'L']:
__data = __data
elif format_ in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=True)
return __data
else:
util_log_info('something wrong with date')
def fetch_index_day(code, __start, __end, format_='numpy', collections=ms.client.flyshare.index_day):
'获取指数日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
if util_date_valid(__end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6], "date_stamp": {
"$lte": util_date_stamp(__end),
"$gte": util_date_stamp(__start)}}):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['vol']), item['date']])
# 多种数据格式
if format_ in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format_ in ['list', 'l', 'L']:
__data = __data
elif format_ in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
return __data
else:
util_log_info('something wrong with date')
def fetch_get_index_min(code, start, end, level='1min', ip=ac.TDX_BEST_IP, port=7709):
'指数分钟线'
api = TdxHq_API()
type_ = ''
if str(level) in ['5', '5m', '5min', 'five']:
level, type_ = 0, '5min'
elif str(level) in ['1', '1m', '1min', 'one']:
level, type_ = 8, '1min'
elif str(level) in ['15', '15m', '15min', 'fifteen']:
level, type_ = 1, '15min'
elif str(level) in ['30', '30m', '30min', 'half']:
level, type_ = 2, '30min'
elif str(level) in ['60', '60m', '60min', '1h']:
level, type_ = 3, '60min'
with api.connect(ip, port):
if str(code)[0] in ['5', '1']: # ETF
data = pd.concat([api.to_df(api.get_security_bars(
level, 1 if str(code)[0] in ['0', '8', '9', '5'] else 0, code, (25 - i) * 800, 800)) for i in range(26)], axis=0)
else:
data = pd.concat([api.to_df(api.get_index_bars(
level, 1 if str(code)[0] in ['0', '8', '9', '5'] else 0, code, (25 - i) * 800, 800)) for i in range(26)], axis=0)
data = data\
.assign(datetime=pd.to_datetime(data['datetime']), code=str(code))\
.drop(['year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=False)\
.assign(date=data['datetime'].apply(lambda x: str(x)[0:10]))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(x)))\
.assign(time_stamp=data['datetime'].apply(lambda x: util_time_stamp(x)))\
.assign(type=type_).set_index('datetime', drop=False, inplace=False)[start:end]
# data
return data.assign(datetime=data['datetime'].apply(lambda x: str(x)))
def fetch_get_index_day(code, start_date, end_date, level='day', ip=ac.TDX_BEST_IP, port=7709):
'指数日线'
api = TdxHq_API()
if level in ['day', 'd', 'D', 'DAY', 'Day']:
level = 9
elif level in ['w', 'W', 'Week', 'week']:
level = 5
elif level in ['month', 'M', 'm', 'Month']:
level = 6
elif level in ['Q', 'Quarter', 'q']:
level = 10
elif level in ['y', 'Y', 'year', 'Year']:
level = 11
with api.connect(ip, port):
if str(code)[0] in ['5', '1']: # ETF
data = pd.concat([api.to_df(api.get_security_bars(
level, 1 if str(code)[0] in ['0', '8', '9', '5'] else 0, code, (25 - i) * 800, 800)) for i in range(26)], axis=0)
else:
data = pd.concat([api.to_df(api.get_index_bars(
level, 1 if str(code)[0] in ['0', '8', '9', '5'] else 0, code, (25 - i) * 800, 800)) for i in range(26)], axis=0)
data = data.assign(date=data['datetime'].apply(lambda x: str(x[0:10]))).assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10])))\
.set_index('date', drop=False, inplace=False)\
.drop(['year', 'month', 'day', 'hour',
'minute', 'datetime'], axis=1)[start_date:end_date]
return data.assign(date=data['date'].apply(lambda x: str(x)[0:10]))
def fetch_stock_day_adv(code,
__start,
__end,
if_drop_index=False,
collections=ms.client.flyshare.stock_day):
'获取股票日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
if isinstance(code, str):
if util_date_valid(__end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6],
"date_stamp": {
"$lte": util_date_stamp(__end),
"$gte": util_date_stamp(__start)
}
}):
__data.append([
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item['vol']),
float(item['amount']), item['date']
])
__data = DataFrame(__data,
columns=[
'code', 'open', 'high', 'low', 'close',
'volume', 'amount', 'date'
])
__data['date'] = pd.to_datetime(__data['date'])
return DataStruct_Stock_day(
__data.query('volume>1').set_index(['date', 'code'],
drop=if_drop_index))
else:
util_log_info('something wrong with date')
elif isinstance(code, list):
return DataStruct_Stock_day(
pd.concat(fetch_stocklist_day(
code, [__start, __end])).query('volume>1').set_index(
['date', 'code'], drop=if_drop_index))
def fetch_get_stock_latest(code, ip=ac.TDX_BEST_IP, port=7709):
code = [code] if isinstance(code, str) else code
api = TdxHq_API(multithread=True)
with api.connect(ip, port):
data = pd.concat([api.to_df(api.get_security_bars(
9, __select_market_code(item), item, 0, 1)).assign(code=item) for item in code], axis=0)
return data\
.assign(date=pd.to_datetime(data['datetime']
.apply(lambda x: x[0:10])), date_stamp=data['datetime']
.apply(lambda x: util_date_stamp(str(x[0:10]))))\
.set_index('date', drop=False)\
.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1)
def SU_save_stock_list(client=ms.client):
data = fetch_get_stock_list()
date = str(datetime.date.today())
date_stamp = util_date_stamp(date)
coll = client.flyshare.stock_list
coll.insert({
'date': date,
'date_stamp': date_stamp,
'stock': {
'code': data
}
})
def SU_update_stock_day(client=ms.client):
data = tushare.fetch_get_stock_list()
date = str(datetime.date.today())
date_stamp = util_date_stamp(date)
#
client.flyshare.drop_collection('stock_list')
client.flyshare.drop_collection('trade_date')
client.flyshare.drop_collection('stock_info')
#client.flyshare.drop_collection('stock_day')
# client.flyshare.user_list.insert(
#{'username': '******', 'password': '******'})
SU_save_stock_info()
SU_save_stock_list()
SU_save_trade_date_all()
#save_stock_day_with_fqfactor()
coll_stocklist = client.flyshare.stock_list
# 使用find_one
stock_list = coll_stocklist.find_one()['stock']['code']
coll_stock_day = client.flyshare.stock_day
stock_list.append('sz50')
stock_list.append('hs300')
for item in stock_list:
util_log_info('updating stock data -- %s' % item)
# coll.find({'code':str(item)[0:6]}).count()
# 先拿到最后一个记录的交易日期
try:
if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = str(
coll_stock_day.find({
'code': str(item)[0:6]
})[coll_stock_day.find({
'code': str(item)[0:6]
}).count() - 1]['date'])
end_date = str(datetime.date.today())
util_log_info('trying updating from %s to %s' %
(start_date, end_date))
data = tushare.fetch_get_stock_day(
str(item)[0:6], start_date, end_date, '02')[1::]
else:
# 这时候直接更新拿到所有的数据就好了
data = tushare.fetch_get_stock_day(item,
startDate='1990-01-01',
if_fq='02')
coll_stock_day.insert_many(data)
except:
util_log_info('error in updating--- %s' % item)
async def fetch_stock_day_adv(
code,
__start,
__end,
if_drop_index=False,
collections=util_sql_async_mongo_setting().flyshare.stock_day):
'获取股票日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
data = [[
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item['vol']), item['date']
] async for item in collections.find({
'code': str(code)[0:6],
"date_stamp": {
"$lte": util_date_stamp(__end),
"$gte": util_date_stamp(__start)
}
})]
def fetch_get_trade_date(endDate, exchange):
data = QATs.trade_cal()
da = data[data.isOpen > 0]
data_json = util_to_json_from_pandas(data)
message = []
for i in range(0, len(data_json) - 1, 1):
date = data_json[i]['calendarDate']
num = i + 1
exchangeName = 'SSE'
data_stamp = util_date_stamp(date)
mes = {
'date': date,
'num': num,
'exchangeName': exchangeName,
'date_stamp': data_stamp
}
message.append(mes)
return message
def save_tdx_to_mongo(file_dir, client=ms.client):
reader = TdxMinBarReader()
__coll = client.flyshare.stock_min_five
for a, v, files in os.walk(file_dir):
for file in files:
if (str(file)[0:2] == 'sh' and int(str(file)[2]) == 6) or \
(str(file)[0:2] == 'sz' and int(str(file)[2]) == 0) or \
(str(file)[0:2] == 'sz' and int(str(file)[2]) == 3):
util_log_info('Now_saving ' + str(file)
[2:8] + '\'s 5 min tick')
fname = file_dir + '\\' + file
df = reader.get_df(fname)
df['code'] = str(file)[2:8]
df['market'] = str(file)[0:2]
df['datetime'] = [str(x) for x in list(df.index)]
df['date'] = [str(x)[0:10] for x in list(df.index)]
df['time_stamp'] = df['datetime'].apply(
lambda x: util_time_stamp(x))
df['date_stamp'] = df['date'].apply(
lambda x: util_date_stamp(x))
data_json = json.loads(df.to_json(orient='records'))
__coll.insert_many(data_json)
def fetch_get_stock_day(name,
startDate='',
endDate='',
if_fq='01',
type_='json'):
if (len(name) != 6):
name = str(name)[0:6]
if str(if_fq) in ['qfq', '01']:
if_fq = 'qfq'
elif str(if_fq) in ['hfq', '02']:
if_fq = 'hfq'
elif str(if_fq) in ['bfq', '00']:
if_fq = 'bfq'
else:
util_log_info('wrong with fq_factor! using qfq')
if_fq = 'qfq'
data = QATs.get_k_data(str(name),
startDate,
endDate,
ktype='D',
autype=if_fq,
retry_count=200,
pause=0.005).sort_index()
data['date_stamp'] = data['date'].apply(lambda x: util_date_stamp(x))
data['fqtype'] = if_fq
if type_ in ['json']:
data_json = util_to_json_from_pandas(data)
return data_json
elif type_ in ['pd', 'pandas', 'p']:
data['date'] = pd.to_datetime(data['date'])
data = data.set_index('date', drop=False)
data['date'] = data['date'].apply(lambda x: str(x)[0:10])
return data
def fetch_get_stock_day(code, start_date, end_date, if_fq='00', level='day', ip=ac.TDX_BEST_IP, port=7709):
api = TdxHq_API()
with api.connect(ip, port):
if level in ['day', 'd', 'D', 'DAY', 'Day']:
level = 9
elif level in ['w', 'W', 'Week', 'week']:
level = 5
elif level in ['month', 'M', 'm', 'Month']:
level = 6
elif level in ['Q', 'Quarter', 'q']:
level = 10
elif level in ['y', 'Y', 'year', 'Year']:
level = 11
data = pd.concat([api.to_df(api.get_security_bars(level, __select_market_code(
code), code, (9 - i) * 800, 800)) for i in range(10)], axis=0)
data = data[data['open'] != 0]
if if_fq in ['00', 'bfq']:
data = data.assign(date=data['datetime'].apply(lambda x: str(x[0:10]))).assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10]))).set_index('date', drop=False, inplace=False)
return data.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1)[start_date:end_date].assign(date=data['date'].apply(lambda x: str(x)[0:10]))
elif if_fq in ['01', 'qfq']:
xdxr_data = fetch_get_stock_xdxr(code)
bfq_data = data.assign(date=pd.to_datetime(data['datetime'].apply(lambda x: str(x[0:10])))).assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10]))).set_index('date', drop=False, inplace=False)
bfq_data = bfq_data.drop(
['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1)
#
if xdxr_data is not None:
info = xdxr_data[xdxr_data['category'] == 1]
bfq_data['if_trade'] = True
data = pd.concat([bfq_data, info[['category']]
[bfq_data.index[0]:]], axis=1)
data['date'] = data.index
data['if_trade'].fillna(value=False, inplace=True)
data = data.fillna(method='ffill')
data = pd.concat([data, info[['fenhong', 'peigu', 'peigujia',
'songzhuangu']][bfq_data.index[0]:]], axis=1)
data = data.fillna(0)
data['preclose'] = (data['close'].shift(1) * 10 - data['fenhong'] + data['peigu']
* data['peigujia']) / (10 + data['peigu'] + data['songzhuangu'])
data['adj'] = (data['preclose'].shift(-1) /
data['close']).fillna(1)[::-1].cumprod()
data['open'] = data['open'] * data['adj']
data['high'] = data['high'] * data['adj']
data['low'] = data['low'] * data['adj']
data['close'] = data['close'] * data['adj']
data['preclose'] = data['preclose'] * data['adj']
data = data[data['if_trade']]
return data.drop(['fenhong', 'peigu', 'peigujia', 'songzhuangu', 'if_trade', 'category'], axis=1)[data['open'] != 0].assign(date=data['date'].apply(lambda x: str(x)[0:10]))[start_date:end_date]
else:
bfq_data['preclose'] = bfq_data['close'].shift(1)
bfq_data['adj'] = 1
return bfq_data[start_date:end_date]
elif if_fq in ['03', 'ddqfq']:
xdxr_data = fetch_get_stock_xdxr(code)
info = xdxr_data[xdxr_data['category'] == 1]
bfq_data = data\
.assign(date=pd.to_datetime(data['datetime'].apply(lambda x: x[0:10])))\
.assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10])))\
.set_index('date', drop=False, inplace=False)\
.drop(['year', 'month', 'day', 'hour',
'minute', 'datetime'], axis=1)
bfq_data['if_trade'] = True
data = pd.concat([bfq_data, info[['category']]
[bfq_data.index[0]:end_date]], axis=1)
data['date'] = data.index
data['if_trade'].fillna(value=False, inplace=True)
data = data.fillna(method='ffill')
data = pd.concat([data, info[['fenhong', 'peigu', 'peigujia',
'songzhuangu']][bfq_data.index[0]:end_date]], axis=1)
data = data.fillna(0)
data['preclose'] = (data['close'].shift(1) * 10 - data['fenhong'] + data['peigu']
* data['peigujia']) / (10 + data['peigu'] + data['songzhuangu'])
data['adj'] = (data['preclose'].shift(-1) /
data['close']).fillna(1)[::-1].cumprod()
data['open'] = data['open'] * data['adj']
data['high'] = data['high'] * data['adj']
data['low'] = data['low'] * data['adj']
data['close'] = data['close'] * data['adj']
data['preclose'] = data['preclose'] * data['adj']
data = data[data['if_trade']]
return data.drop(['fenhong', 'peigu', 'peigujia', 'songzhuangu', 'if_trade', 'category'], axis=1)[data['open'] != 0].assign(date=data['date'].apply(lambda x: str(x)[0:10]))[start_date:end_date]
elif if_fq in ['02', 'hfq']:
xdxr_data = fetch_get_stock_xdxr(code)
info = xdxr_data[xdxr_data['category'] == 1]
bfq_data = data\
.assign(date=pd.to_datetime(data['datetime'].apply(lambda x: x[0:10])))\
.assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10])))\
.set_index('date', drop=False, inplace=False)\
.drop(['year', 'month', 'day', 'hour',
'minute', 'datetime'], axis=1)
bfq_data['if_trade'] = True
data = pd.concat([bfq_data, info[['category']]
[bfq_data.index[0]:]], axis=1)
data['date'] = data.index
data['if_trade'].fillna(value=False, inplace=True)
data = data.fillna(method='ffill')
data = pd.concat([data, info[['fenhong', 'peigu', 'peigujia',
'songzhuangu']][bfq_data.index[0]:]], axis=1)
data = data.fillna(0)
data['preclose'] = (data['close'].shift(1) * 10 - data['fenhong'] + data['peigu']
* data['peigujia']) / (10 + data['peigu'] + data['songzhuangu'])
data['adj'] = (data['preclose'].shift(-1) /
data['close']).fillna(1).cumprod()
data['open'] = data['open'] / data['adj']
data['high'] = data['high'] / data['adj']
data['low'] = data['low'] / data['adj']
data['close'] = data['close'] / data['adj']
data['preclose'] = data['preclose'] / data['adj']
data = data[data['if_trade']]
return data.drop(['fenhong', 'peigu', 'peigujia', 'songzhuangu', 'if_trade', 'category'], axis=1)[data['open'] != 0].assign(date=data['date'].apply(lambda x: str(x)[0:10]))[start_date:end_date]
elif if_fq in ['04', 'ddhfq']:
xdxr_data = fetch_get_stock_xdxr(code)
info = xdxr_data[xdxr_data['category'] == 1]
bfq_data = data\
.assign(date=pd.to_datetime(data['datetime'].apply(lambda x: x[0:10])))\
.assign(code=str(code))\
.assign(date_stamp=data['datetime'].apply(lambda x: util_date_stamp(str(x)[0:10])))\
.set_index('date', drop=False, inplace=False)\
.drop(['year', 'month', 'day', 'hour',
'minute', 'datetime'], axis=1)
bfq_data['if_trade'] = True
data = pd.concat([bfq_data, info[['category']]
[bfq_data.index[0]:end_date]], axis=1)
data['date'] = data.index
data['if_trade'].fillna(value=False, inplace=True)
data = data.fillna(method='ffill')
data = pd.concat([data, info[['fenhong', 'peigu', 'peigujia',
'songzhuangu']][bfq_data.index[0]:end_date]], axis=1)
data = data.fillna(0)
data['preclose'] = (data['close'].shift(1) * 10 - data['fenhong'] + data['peigu']
* data['peigujia']) / (10 + data['peigu'] + data['songzhuangu'])
data['adj'] = (data['preclose'].shift(-1) /
data['close']).fillna(1).cumprod()
data['open'] = data['open'] / data['adj']
data['high'] = data['high'] / data['adj']
data['low'] = data['low'] / data['adj']
data['close'] = data['close'] / data['adj']
data['preclose'] = data['preclose'] / data['adj']
data = data[data['if_trade']]
return data.drop(['fenhong', 'peigu', 'peigujia', 'songzhuangu', 'if_trade', 'category'], axis=1)[data['open'] != 0].assign(date=data['date'].apply(lambda x: str(x)[0:10]))[start_date:end_date]