def QA_update_stock_day_all(client):
coll_stocklist=client.quantaxis.stock_list
stock_list=coll_stocklist.find_one()['stock']['code']
coll_stock_day=client.quantaxis.stock_day
for item in stock_list:
QA_util_log_info('updating stock data -- %s'%item)
#coll.find({'code':str(item)[0:6]}).count()
#先拿到最后一个记录的交易日期
try:
if coll_stock_day.find({'code':str(item)[0:6]}).count()>0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date=str(coll_stock_day.find({'code':str(item)[0:6]})[coll_stock_day.find({'code':str(item)[0:6]}).count()-1]['date'])
end_date=str(datetime.date.today())
QA_util_log_info('trying updating from %s to %s' %(start_date,end_date))
data=QATushare.QA_fetch_get_stock_day(str(item)[0:6],start_date,end_date)[1::]
else:
# 这时候直接更新拿到所有的数据就好了
data=QATushare.QA_fetch_get_stock_day(str(item)[0:6])
coll_stock_day.insert_many(data)
except:
QA_util_log_info('error in updating--- %s'%item)
def QA_SU_update_stock_day(client=QA_Setting.client):
data = QATushare.QA_fetch_get_stock_list()
date = str(datetime.date.today())
date_stamp = QA_util_date_stamp(date)
#
client.quantaxis.drop_collection('stock_list')
client.quantaxis.drop_collection('trade_date')
client.quantaxis.drop_collection('stock_info')
#client.quantaxis.drop_collection('stock_day')
# client.quantaxis.user_list.insert(
#{'username': '******', 'password': '******'})
QA_SU_save_stock_info()
QA_SU_save_stock_list()
QA_SU_save_trade_date_all()
#QA_save_stock_day_with_fqfactor()
coll_stocklist = client.quantaxis.stock_list
# 使用find_one
stock_list = coll_stocklist.find_one()['stock']['code']
coll_stock_day = client.quantaxis.stock_day
stock_list.append('sz50')
stock_list.append('hs300')
for item in stock_list:
QA_util_log_info('updating stock data -- %s' % item)
# coll.find({'code':str(item)[0:6]}).count()
# 先拿到最后一个记录的交易日期
try:
if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = str(
coll_stock_day.find({
'code': str(item)[0:6]
})[coll_stock_day.find({
'code': str(item)[0:6]
}).count() - 1]['date'])
end_date = str(datetime.date.today())
QA_util_log_info('trying updating from %s to %s' %
(start_date, end_date))
data = QATushare.QA_fetch_get_stock_day(
str(item)[0:6], start_date, end_date, '02')[1::]
else:
# 这时候直接更新拿到所有的数据就好了
data = QATushare.QA_fetch_get_stock_day(item,
startDate='1990-01-01',
if_fq='02')
coll_stock_day.insert_many(data)
except:
QA_util_log_info('error in updating--- %s' % item)
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_bfq = QATushare.QA_fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='00',
type_='pd')
data_qfq = QATushare.QA_fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='01',
type_='pd')
data_qfq['qfqfactor'] = data_qfq['open'] / data_bfq['open']
data_json = QA_util_to_json_from_pandas(data_qfq)
__coll.insert_many(data_json)
except:
QA_util_log_info('error in saving ==== %s' % str(i))
def QA_SU_save_stock_list(client):
data = QATushare.QA_fetch_get_stock_list()
date = str(datetime.date.today())
date_stamp = QA_util_date_stamp(date)
coll = client.quantaxis.stock_list
coll.insert({'date': date, 'date_stamp': date_stamp,
'stock': {'code': data}})
def saving_work(i):
print('Now Saving ==== %s' % (i))
try:
data_json = QATushare.QA_fetch_get_stock_day(i)
__coll.insert_many(data_json)
except:
print('error in saving ==== %s' % str(i))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_json = QATushare.QA_fetch_get_stock_day(
i, startDate='1990-01-01', if_fq='00')
__coll.insert_many(data_json)
except:
QA_util_log_info('error in saving ==== %s' % str(i))
def QA_update_stock_day_all(code,client):
coll_stocklist=client.quantaxis.stock_list
stock_list=coll_stocklist.find_one()['stock']['code']
coll_stock_day=client.quantaxis.stock_day
for item in stock_list:
#coll.find({'code':str(item)[0:6]}).count()
#先拿到最后一个记录的交易日期
start_date=coll_stock_day.find({'code':str(item)[0:6]})[coll_stock_day.find({'code':str(item)[0:6]}).count()-1]['date']
end_date=str(datetime.date.today())
data=QATushare.QA_fetch_get_stock_day(str(item)[0:6],start_date,end_date)[1::]
coll_stock_day.insert_many(data)
def QA_quotation(code, start, end, frequence, market, source, output):
"""一个统一的fetch
Arguments:
code {str/list} -- 证券/股票的代码
start {str} -- 开始日期
end {str} -- 结束日期
frequence {enum} -- 频率 QA.FREQUENCE
market {enum} -- 市场 QA.MARKET_TYPE
source {enum} -- 来源 QA.DATASOURCE
output {enum} -- 输出类型 QA.OUTPUT_FORMAT
"""
if market is MARKET_TYPE.STOCK_CN:
if frequence is FREQUENCE.DAY:
if source is DATASOURCE.MONGO:
res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end)
elif source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_day(code, start, end, '00')
elif source is DATASOURCE.TUSHARE:
res = QATushare.QA_fetch_get_stock_day(code, start, end, '00')
elif frequence in [
FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN,
FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN
]:
if source is DATASOURCE.MONGO:
res = QAQueryAdv.QA_fetch_stock_min_adv(code,
start,
end,
frequence=frequence)
elif source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_min(code,
start,
end,
frequence=frequence)
elif frequence is FREQUENCE.TICK:
if source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_transaction(code, start, end)
# 指数代码和股票代码是冲突重复的， sh000001 上证指数 000001 是不同的
elif market is MARKET_TYPE.INDEX_CN:
if frequence is FREQUENCE.DAY:
if source is DATASOURCE.MONGO:
res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end)
elif market is MARKET_TYPE.OPTION_CN:
if source is DATASOURCE.MONGO:
#res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end)
raise NotImplementedError('CURRENT NOT FINISH THIS METHOD')
# print(type(res))
return res
def QA_update_stock_day_all(client=QA_Setting.client):
data = QATushare.QA_fetch_get_stock_list()
date = str(datetime.date.today())
date_stamp = QA_util_date_stamp(date)
#
client.quantaxis.drop_collection('stock_list')
client.quantaxis.drop_collection('trade_date')
client.quantaxis.drop_collection('stock_info')
client.quantaxis.drop_collection('stock_day')
# client.quantaxis.user_list.insert(
#{'username': '******', 'password': '******'})
QA_SU_save_stock_info()
QA_SU_save_stock_list()
QA_SU_save_trade_date_all()
QA_save_stock_day_with_fqfactor()
"""
def QA_quotation(code, start, end, frequence, market, source, output):
"""一个统一的fetch
Arguments:
code {str/list} -- 证券/股票的代码
start {str} -- 开始日期
end {str} -- 结束日期
frequence {enum} -- 频率 QA.FREQUENCE
market {enum} -- 市场 QA.MARKET_TYPE
source {enum} -- 来源 QA.DATASOURCE
output {enum} -- 输出类型 QA.OUTPUT_FORMAT
"""
if market is MARKET_TYPE.STOCK_CN:
if frequence is FREQUENCE.DAY:
if source is DATASOURCE.MONGO:
res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end)
elif source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_day(code, start, end, '00')
elif source is DATASOURCE.TUSHARE:
res = QATushare.QA_fetch_get_stock_day(code, start, end, '00')
elif frequence in [
FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN,
FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN
]:
if source is DATASOURCE.MONGO:
res = QAQueryAdv.QA_fetch_stock_min_adv(code,
start,
end,
frequence=frequence)
elif source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_min(code,
start,
end,
frequence=frequence)
elif frequence is FREQUENCE.TICK:
if source is DATASOURCE.TDX:
res = QATdx.QA_fetch_get_stock_transaction(code, start, end)
print(type(res))
return res
def QA_update_stock_day_all(client):
coll_stocklist = client.quantaxis.stock_list
stock_list = coll_stocklist.find_one()['stock']['code']
coll_stock_day = client.quantaxis.stock_day
for item in stock_list:
QA_util_log_info('updating stock data -- %s' % item)
#coll.find({'code':str(item)[0:6]}).count()
#先拿到最后一个记录的交易日期
start_date = str(
coll_stock_day.find({'code': str(item)[0:6]
})[coll_stock_day.find({
'code': str(item)[0:6]
}).count() - 1]['date'])
end_date = str(datetime.date.today())
QA_util_log_info('trying updating from %s to %s' %
(start_date, end_date))
data = QATushare.QA_fetch_get_stock_day(
str(item)[0:6], start_date, end_date)[1::]
try:
coll_stock_day.insert_many(data)
except:
QA_util_log_info('error in updating--- %s' % item)
def QA_SU_save_trade_date_all(client=QA_Setting.client):
data = QATushare.QA_fetch_get_trade_date('', '')
coll = client.quantaxis.trade_date
coll.insert_many(data)
def QA_SU_save_stock_info(client):
data = QATushare.QA_fetch_get_stock_info('all')
coll = client.quantaxis.stock_info
coll.insert_many(data)
def QA_SU_save_trade_date_all():
data = QATushare.QA_fetch_get_trade_date('', '')
coll = pymongo.MongoClient().quantaxis.trade_date
coll.insert_many(data)
def QA_quotation(code, start, end, frequence, market, source=DATASOURCE.TDX, output=OUTPUT_FORMAT.DATAFRAME):
"""一个统一的获取k线的方法
如果使用mongo,从本地数据库获取,失败则在线获取
Arguments:
code {str/list} -- 期货/股票的代码
start {str} -- 开始日期
end {str} -- 结束日期
frequence {enum} -- 频率 QA.FREQUENCE
market {enum} -- 市场 QA.MARKET_TYPE
source {enum} -- 来源 QA.DATASOURCE
output {enum} -- 输出类型 QA.OUTPUT_FORMAT
"""
res = None
if market == MARKET_TYPE.STOCK_CN:
if frequence == FREQUENCE.DAY:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_stock_day(code, start, end, '00')
res = QA_DataStruct_Stock_day(res.set_index(['date', 'code']))
elif source == DATASOURCE.TUSHARE:
res = QATushare.QA_fetch_get_stock_day(code, start, end, '00')
elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_stock_min_adv(
code, start, end, frequence=frequence)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_stock_min(
code, start, end, frequence=frequence)
res = QA_DataStruct_Stock_min(
res.set_index(['datetime', 'code']))
elif market == MARKET_TYPE.FUTURE_CN:
if frequence == FREQUENCE.DAY:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_future_day_adv(code, start, end)
except:
res = None
if source == DATASOURCE.TDX or res is None:
res = QATdx.QA_fetch_get_future_day(code, start, end)
res = QA_DataStruct_Future_day(res.set_index(['date', 'code']))
elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_future_min_adv(
code, start, end, frequence=frequence)
except:
res = None
if source == DATASOURCE.TDX or res is None:
res = QATdx.QA_fetch_get_future_min(
code, start, end, frequence=frequence)
res = QA_DataStruct_Future_min(
res.set_index(['datetime', 'code']))
elif market == MARKET_TYPE.INDEX_CN:
if frequence == FREQUENCE.DAY:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end)
except:
return None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_index_day(code, start, end)
res = QA_DataStruct_Index_day(res.set_index(['date', 'code']))
elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_index_min_adv(
code, start, end, frequence=frequence)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_index_min(
code, start, end, frequence=frequence)
res = QA_DataStruct_Index_min(
res.set_index(['datetime', 'code']))
elif market == MARKET_TYPE.OPTION_CN:
if source == DATASOURCE.MONGO:
#res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end)
raise NotImplementedError('CURRENT NOT FINISH THIS METHOD')
# print(type(res))
if output is OUTPUT_FORMAT.DATAFRAME:
return res.data
elif output is OUTPUT_FORMAT.DATASTRUCT:
return res
elif output is OUTPUT_FORMAT.NDARRAY:
return res.to_numpy()
elif output is OUTPUT_FORMAT.JSON:
return res.to_json()
elif output is OUTPUT_FORMAT.LIST:
return res.to_list()
def QA_quotation_adv(code, start, end=save_tdx.now_time(), frequence='1min',
market=MARKET_TYPE.STOCK_CN, source=DATASOURCE.AUTO, output=OUTPUT_FORMAT.DATAFRAME):
"""一个统一的获取k线的方法
如果source=DATASOURCE.AUTO,优先mongo,从本地数据库获取,mongo中未下载的数据从TDX中在线补全。(仅限股票)
Arguments:
code {str/list} -- 期货/股票的代码
start {str} -- 开始日期
end {str} -- 结束日期
frequence {enum} -- 频率 QA.FREQUENCE
market {enum} -- 市场 QA.MARKET_TYPE
source {enum} -- 来源 QA.DATASOURCE
output {enum} -- 输出类型 QA.OUTPUT_FORMAT
"""
if pd.Timestamp(end) > pd.Timestamp(save_tdx.now_time()):
end = save_tdx.now_time()
res = None
if market == MARKET_TYPE.STOCK_CN:
if frequence == FREQUENCE.DAY or frequence == FREQUENCE.WEEK:
if source == DATASOURCE.AUTO:
try:
# 返回的是QA_DataStruct_Stock_day对象，为了与在线获取的数据格式保持统一，转成单索引
res = QAQueryAdv.QA_fetch_stock_day_adv(
code, start, end).data.reset_index(level='code')
# res = QAQueryAdv.QA_fetch_stock_day_adv(
# code, start, end).data.reset_index(level='code')[:14]
start_date = res.index[-1]
end_date = pd.Timestamp(end)
if end_date-start_date > datetime.timedelta(hours=17):
# 从TDX补充数据，由于仅考虑个股，在这里不做入库操作，入库还是需要save
data_tdx = QATdx.QA_fetch_get_stock_day(
code, QA_util_get_next_period(start_date, frequence), end_date, '00')
# data_tdx与从数据库获取的数据格式上做一些统一。
data_tdx = data_tdx.rename(columns={"vol": "volume"}).drop([
'date', 'date_stamp'], axis=1)
data_tdx.index = pd.to_datetime(data_tdx.index)
res = pd.concat([res, data_tdx], sort=True)
res = QA_DataStruct_Stock_day(
res.reset_index().set_index(['date', 'code']))
except:
res = None
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_stock_day(code, start, end, '00')
res = QA_DataStruct_Stock_day(res.set_index(['date', 'code']))
elif source == DATASOURCE.TUSHARE:
res = QATushare.QA_fetch_get_stock_day(code, start, end, '00')
if frequence == FREQUENCE.WEEK:
res = QA_DataStruct_Stock_day(
QA_data_day_resample(res.data))
elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.AUTO:
try:
# 返回的是QA_DataStruct_Stock_day对象，为了与在线获取的数据格式保持统一，转成单索引
res = QAQueryAdv.QA_fetch_stock_min_adv(
code, start, end, frequence=frequence).data.reset_index(level='code')
# res = QAQueryAdv.QA_fetch_stock_min_adv(
# code, start, end, frequence=frequence).data.reset_index(level='code')[:710]
start_date = res.index[-1]
end_date = pd.Timestamp(end)
if end_date > start_date:
# 从TDX补充数据，由于仅考虑个股，在这里不做入库操作，入库还是需要save
data_tdx = QATdx.QA_fetch_get_stock_min(code, QA_util_get_next_period(
start_date, frequence), end_date, frequence=frequence)
# data_tdx与从数据库获取的数据格式上做一些统一。
data_tdx = data_tdx.rename(columns={"vol": "volume"}).drop(
['date', 'datetime', 'date_stamp', 'time_stamp'], axis=1)
data_tdx.index = pd.to_datetime(data_tdx.index)
res = pd.concat([res, data_tdx], sort=True)
res = QA_DataStruct_Stock_day(
res.reset_index().set_index(['datetime', 'code']))
except:
res = None
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_stock_min_adv(
code,
start,
end,
frequence=frequence
)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_stock_min(
code,
start,
end,
frequence=frequence
)
res = QA_DataStruct_Stock_min(
res.set_index(['datetime',
'code'])
)
elif market == MARKET_TYPE.FUTURE_CN:
if frequence == FREQUENCE.DAY:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_future_day_adv(code, start, end)
except:
res = None
if source == DATASOURCE.TDX or res is None:
res = QATdx.QA_fetch_get_future_day(code, start, end)
res = QA_DataStruct_Future_day(res.set_index(['date', 'code']))
elif frequence in [FREQUENCE.ONE_MIN,
FREQUENCE.FIVE_MIN,
FREQUENCE.FIFTEEN_MIN,
FREQUENCE.THIRTY_MIN,
FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_future_min_adv(
code,
start,
end,
frequence=frequence
)
except:
res = None
if source == DATASOURCE.TDX or res is None:
res = QATdx.QA_fetch_get_future_min(
code,
start,
end,
frequence=frequence
)
res = QA_DataStruct_Future_min(
res.set_index(['datetime',
'code'])
)
elif market == MARKET_TYPE.INDEX_CN:
if frequence == FREQUENCE.DAY:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end)
except:
return None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_index_day(code, start, end)
res = QA_DataStruct_Index_day(res.set_index(['date', 'code']))
elif frequence in [FREQUENCE.ONE_MIN,
FREQUENCE.FIVE_MIN,
FREQUENCE.FIFTEEN_MIN,
FREQUENCE.THIRTY_MIN,
FREQUENCE.SIXTY_MIN]:
if source == DATASOURCE.MONGO:
try:
res = QAQueryAdv.QA_fetch_index_min_adv(
code,
start,
end,
frequence=frequence
)
except:
res = None
if source == DATASOURCE.TDX or res == None:
res = QATdx.QA_fetch_get_index_min(
code,
start,
end,
frequence=frequence
)
res = QA_DataStruct_Index_min(
res.set_index(['datetime',
'code'])
)
elif market == MARKET_TYPE.OPTION_CN:
if source == DATASOURCE.MONGO:
#res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end)
raise NotImplementedError('CURRENT NOT FINISH THIS METHOD')
# print(type(res))
if output is OUTPUT_FORMAT.DATAFRAME:
return res.data
elif output is OUTPUT_FORMAT.DATASTRUCT:
return res
elif output is OUTPUT_FORMAT.NDARRAY:
return res.to_numpy()
elif output is OUTPUT_FORMAT.JSON:
return res.to_json()
elif output is OUTPUT_FORMAT.LIST:
return res.to_list()
def QA_update_stock_day(name, startDate, endDate):
data = QATushare.QA_fetch_get_stock_day(name, startDate, endDate)