def QA_update_stock_day_all(client): coll_stocklist=client.quantaxis.stock_list stock_list=coll_stocklist.find_one()['stock']['code'] coll_stock_day=client.quantaxis.stock_day for item in stock_list: QA_util_log_info('updating stock data -- %s'%item) #coll.find({'code':str(item)[0:6]}).count() #先拿到最后一个记录的交易日期 try: if coll_stock_day.find({'code':str(item)[0:6]}).count()>0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date=str(coll_stock_day.find({'code':str(item)[0:6]})[coll_stock_day.find({'code':str(item)[0:6]}).count()-1]['date']) end_date=str(datetime.date.today()) QA_util_log_info('trying updating from %s to %s' %(start_date,end_date)) data=QATushare.QA_fetch_get_stock_day(str(item)[0:6],start_date,end_date)[1::] else: # 这时候直接更新拿到所有的数据就好了 data=QATushare.QA_fetch_get_stock_day(str(item)[0:6]) coll_stock_day.insert_many(data) except: QA_util_log_info('error in updating--- %s'%item)
def QA_SU_update_stock_day(client=QA_Setting.client): data = QATushare.QA_fetch_get_stock_list() date = str(datetime.date.today()) date_stamp = QA_util_date_stamp(date) # client.quantaxis.drop_collection('stock_list') client.quantaxis.drop_collection('trade_date') client.quantaxis.drop_collection('stock_info') #client.quantaxis.drop_collection('stock_day') # client.quantaxis.user_list.insert( #{'username': '******', 'password': '******'}) QA_SU_save_stock_info() QA_SU_save_stock_list() QA_SU_save_trade_date_all() #QA_save_stock_day_with_fqfactor() coll_stocklist = client.quantaxis.stock_list # 使用find_one stock_list = coll_stocklist.find_one()['stock']['code'] coll_stock_day = client.quantaxis.stock_day stock_list.append('sz50') stock_list.append('hs300') for item in stock_list: QA_util_log_info('updating stock data -- %s' % item) # coll.find({'code':str(item)[0:6]}).count() # 先拿到最后一个记录的交易日期 try: if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = str( coll_stock_day.find({ 'code': str(item)[0:6] })[coll_stock_day.find({ 'code': str(item)[0:6] }).count() - 1]['date']) end_date = str(datetime.date.today()) QA_util_log_info('trying updating from %s to %s' % (start_date, end_date)) data = QATushare.QA_fetch_get_stock_day( str(item)[0:6], start_date, end_date, '02')[1::] else: # 这时候直接更新拿到所有的数据就好了 data = QATushare.QA_fetch_get_stock_day(item, startDate='1990-01-01', if_fq='02') coll_stock_day.insert_many(data) except: QA_util_log_info('error in updating--- %s' % item)
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_bfq = QATushare.QA_fetch_get_stock_day(i, startDate='1990-01-01', if_fq='00', type_='pd') data_qfq = QATushare.QA_fetch_get_stock_day(i, startDate='1990-01-01', if_fq='01', type_='pd') data_qfq['qfqfactor'] = data_qfq['open'] / data_bfq['open'] data_json = QA_util_to_json_from_pandas(data_qfq) __coll.insert_many(data_json) except: QA_util_log_info('error in saving ==== %s' % str(i))
def QA_SU_save_stock_list(client): data = QATushare.QA_fetch_get_stock_list() date = str(datetime.date.today()) date_stamp = QA_util_date_stamp(date) coll = client.quantaxis.stock_list coll.insert({'date': date, 'date_stamp': date_stamp, 'stock': {'code': data}})
def saving_work(i): print('Now Saving ==== %s' % (i)) try: data_json = QATushare.QA_fetch_get_stock_day(i) __coll.insert_many(data_json) except: print('error in saving ==== %s' % str(i))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_json = QATushare.QA_fetch_get_stock_day( i, startDate='1990-01-01', if_fq='00') __coll.insert_many(data_json) except: QA_util_log_info('error in saving ==== %s' % str(i))
def QA_update_stock_day_all(code,client): coll_stocklist=client.quantaxis.stock_list stock_list=coll_stocklist.find_one()['stock']['code'] coll_stock_day=client.quantaxis.stock_day for item in stock_list: #coll.find({'code':str(item)[0:6]}).count() #先拿到最后一个记录的交易日期 start_date=coll_stock_day.find({'code':str(item)[0:6]})[coll_stock_day.find({'code':str(item)[0:6]}).count()-1]['date'] end_date=str(datetime.date.today()) data=QATushare.QA_fetch_get_stock_day(str(item)[0:6],start_date,end_date)[1::] coll_stock_day.insert_many(data)
def QA_quotation(code, start, end, frequence, market, source, output): """一个统一的fetch Arguments: code {str/list} -- 证券/股票的代码 start {str} -- 开始日期 end {str} -- 结束日期 frequence {enum} -- 频率 QA.FREQUENCE market {enum} -- 市场 QA.MARKET_TYPE source {enum} -- 来源 QA.DATASOURCE output {enum} -- 输出类型 QA.OUTPUT_FORMAT """ if market is MARKET_TYPE.STOCK_CN: if frequence is FREQUENCE.DAY: if source is DATASOURCE.MONGO: res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end) elif source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_day(code, start, end, '00') elif source is DATASOURCE.TUSHARE: res = QATushare.QA_fetch_get_stock_day(code, start, end, '00') elif frequence in [ FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN ]: if source is DATASOURCE.MONGO: res = QAQueryAdv.QA_fetch_stock_min_adv(code, start, end, frequence=frequence) elif source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_min(code, start, end, frequence=frequence) elif frequence is FREQUENCE.TICK: if source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_transaction(code, start, end) # 指数代码和股票代码是冲突重复的, sh000001 上证指数 000001 是不同的 elif market is MARKET_TYPE.INDEX_CN: if frequence is FREQUENCE.DAY: if source is DATASOURCE.MONGO: res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end) elif market is MARKET_TYPE.OPTION_CN: if source is DATASOURCE.MONGO: #res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end) raise NotImplementedError('CURRENT NOT FINISH THIS METHOD') # print(type(res)) return res
def QA_update_stock_day_all(client=QA_Setting.client): data = QATushare.QA_fetch_get_stock_list() date = str(datetime.date.today()) date_stamp = QA_util_date_stamp(date) # client.quantaxis.drop_collection('stock_list') client.quantaxis.drop_collection('trade_date') client.quantaxis.drop_collection('stock_info') client.quantaxis.drop_collection('stock_day') # client.quantaxis.user_list.insert( #{'username': '******', 'password': '******'}) QA_SU_save_stock_info() QA_SU_save_stock_list() QA_SU_save_trade_date_all() QA_save_stock_day_with_fqfactor() """
def QA_quotation(code, start, end, frequence, market, source, output): """一个统一的fetch Arguments: code {str/list} -- 证券/股票的代码 start {str} -- 开始日期 end {str} -- 结束日期 frequence {enum} -- 频率 QA.FREQUENCE market {enum} -- 市场 QA.MARKET_TYPE source {enum} -- 来源 QA.DATASOURCE output {enum} -- 输出类型 QA.OUTPUT_FORMAT """ if market is MARKET_TYPE.STOCK_CN: if frequence is FREQUENCE.DAY: if source is DATASOURCE.MONGO: res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end) elif source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_day(code, start, end, '00') elif source is DATASOURCE.TUSHARE: res = QATushare.QA_fetch_get_stock_day(code, start, end, '00') elif frequence in [ FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN ]: if source is DATASOURCE.MONGO: res = QAQueryAdv.QA_fetch_stock_min_adv(code, start, end, frequence=frequence) elif source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_min(code, start, end, frequence=frequence) elif frequence is FREQUENCE.TICK: if source is DATASOURCE.TDX: res = QATdx.QA_fetch_get_stock_transaction(code, start, end) print(type(res)) return res
def QA_update_stock_day_all(client): coll_stocklist = client.quantaxis.stock_list stock_list = coll_stocklist.find_one()['stock']['code'] coll_stock_day = client.quantaxis.stock_day for item in stock_list: QA_util_log_info('updating stock data -- %s' % item) #coll.find({'code':str(item)[0:6]}).count() #先拿到最后一个记录的交易日期 start_date = str( coll_stock_day.find({'code': str(item)[0:6] })[coll_stock_day.find({ 'code': str(item)[0:6] }).count() - 1]['date']) end_date = str(datetime.date.today()) QA_util_log_info('trying updating from %s to %s' % (start_date, end_date)) data = QATushare.QA_fetch_get_stock_day( str(item)[0:6], start_date, end_date)[1::] try: coll_stock_day.insert_many(data) except: QA_util_log_info('error in updating--- %s' % item)
def QA_SU_save_trade_date_all(client=QA_Setting.client): data = QATushare.QA_fetch_get_trade_date('', '') coll = client.quantaxis.trade_date coll.insert_many(data)
def QA_SU_save_stock_info(client): data = QATushare.QA_fetch_get_stock_info('all') coll = client.quantaxis.stock_info coll.insert_many(data)
def QA_SU_save_trade_date_all(): data = QATushare.QA_fetch_get_trade_date('', '') coll = pymongo.MongoClient().quantaxis.trade_date coll.insert_many(data)
def QA_quotation(code, start, end, frequence, market, source=DATASOURCE.TDX, output=OUTPUT_FORMAT.DATAFRAME): """一个统一的获取k线的方法 如果使用mongo,从本地数据库获取,失败则在线获取 Arguments: code {str/list} -- 期货/股票的代码 start {str} -- 开始日期 end {str} -- 结束日期 frequence {enum} -- 频率 QA.FREQUENCE market {enum} -- 市场 QA.MARKET_TYPE source {enum} -- 来源 QA.DATASOURCE output {enum} -- 输出类型 QA.OUTPUT_FORMAT """ res = None if market == MARKET_TYPE.STOCK_CN: if frequence == FREQUENCE.DAY: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_stock_day(code, start, end, '00') res = QA_DataStruct_Stock_day(res.set_index(['date', 'code'])) elif source == DATASOURCE.TUSHARE: res = QATushare.QA_fetch_get_stock_day(code, start, end, '00') elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_stock_min_adv( code, start, end, frequence=frequence) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_stock_min( code, start, end, frequence=frequence) res = QA_DataStruct_Stock_min( res.set_index(['datetime', 'code'])) elif market == MARKET_TYPE.FUTURE_CN: if frequence == FREQUENCE.DAY: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_future_day_adv(code, start, end) except: res = None if source == DATASOURCE.TDX or res is None: res = QATdx.QA_fetch_get_future_day(code, start, end) res = QA_DataStruct_Future_day(res.set_index(['date', 'code'])) elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_future_min_adv( code, start, end, frequence=frequence) except: res = None if source == DATASOURCE.TDX or res is None: res = QATdx.QA_fetch_get_future_min( code, start, end, frequence=frequence) res = QA_DataStruct_Future_min( res.set_index(['datetime', 'code'])) elif market == MARKET_TYPE.INDEX_CN: if frequence == FREQUENCE.DAY: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end) except: return None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_index_day(code, start, end) res = QA_DataStruct_Index_day(res.set_index(['date', 'code'])) elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_index_min_adv( code, start, end, frequence=frequence) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_index_min( code, start, end, frequence=frequence) res = QA_DataStruct_Index_min( res.set_index(['datetime', 'code'])) elif market == MARKET_TYPE.OPTION_CN: if source == DATASOURCE.MONGO: #res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end) raise NotImplementedError('CURRENT NOT FINISH THIS METHOD') # print(type(res)) if output is OUTPUT_FORMAT.DATAFRAME: return res.data elif output is OUTPUT_FORMAT.DATASTRUCT: return res elif output is OUTPUT_FORMAT.NDARRAY: return res.to_numpy() elif output is OUTPUT_FORMAT.JSON: return res.to_json() elif output is OUTPUT_FORMAT.LIST: return res.to_list()
def QA_quotation_adv(code, start, end=save_tdx.now_time(), frequence='1min', market=MARKET_TYPE.STOCK_CN, source=DATASOURCE.AUTO, output=OUTPUT_FORMAT.DATAFRAME): """一个统一的获取k线的方法 如果source=DATASOURCE.AUTO,优先mongo,从本地数据库获取,mongo中未下载的数据从TDX中在线补全。(仅限股票) Arguments: code {str/list} -- 期货/股票的代码 start {str} -- 开始日期 end {str} -- 结束日期 frequence {enum} -- 频率 QA.FREQUENCE market {enum} -- 市场 QA.MARKET_TYPE source {enum} -- 来源 QA.DATASOURCE output {enum} -- 输出类型 QA.OUTPUT_FORMAT """ if pd.Timestamp(end) > pd.Timestamp(save_tdx.now_time()): end = save_tdx.now_time() res = None if market == MARKET_TYPE.STOCK_CN: if frequence == FREQUENCE.DAY or frequence == FREQUENCE.WEEK: if source == DATASOURCE.AUTO: try: # 返回的是QA_DataStruct_Stock_day对象,为了与在线获取的数据格式保持统一,转成单索引 res = QAQueryAdv.QA_fetch_stock_day_adv( code, start, end).data.reset_index(level='code') # res = QAQueryAdv.QA_fetch_stock_day_adv( # code, start, end).data.reset_index(level='code')[:14] start_date = res.index[-1] end_date = pd.Timestamp(end) if end_date-start_date > datetime.timedelta(hours=17): # 从TDX补充数据,由于仅考虑个股,在这里不做入库操作,入库还是需要save data_tdx = QATdx.QA_fetch_get_stock_day( code, QA_util_get_next_period(start_date, frequence), end_date, '00') # data_tdx与从数据库获取的数据格式上做一些统一。 data_tdx = data_tdx.rename(columns={"vol": "volume"}).drop([ 'date', 'date_stamp'], axis=1) data_tdx.index = pd.to_datetime(data_tdx.index) res = pd.concat([res, data_tdx], sort=True) res = QA_DataStruct_Stock_day( res.reset_index().set_index(['date', 'code'])) except: res = None if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_stock_day_adv(code, start, end) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_stock_day(code, start, end, '00') res = QA_DataStruct_Stock_day(res.set_index(['date', 'code'])) elif source == DATASOURCE.TUSHARE: res = QATushare.QA_fetch_get_stock_day(code, start, end, '00') if frequence == FREQUENCE.WEEK: res = QA_DataStruct_Stock_day( QA_data_day_resample(res.data)) elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.AUTO: try: # 返回的是QA_DataStruct_Stock_day对象,为了与在线获取的数据格式保持统一,转成单索引 res = QAQueryAdv.QA_fetch_stock_min_adv( code, start, end, frequence=frequence).data.reset_index(level='code') # res = QAQueryAdv.QA_fetch_stock_min_adv( # code, start, end, frequence=frequence).data.reset_index(level='code')[:710] start_date = res.index[-1] end_date = pd.Timestamp(end) if end_date > start_date: # 从TDX补充数据,由于仅考虑个股,在这里不做入库操作,入库还是需要save data_tdx = QATdx.QA_fetch_get_stock_min(code, QA_util_get_next_period( start_date, frequence), end_date, frequence=frequence) # data_tdx与从数据库获取的数据格式上做一些统一。 data_tdx = data_tdx.rename(columns={"vol": "volume"}).drop( ['date', 'datetime', 'date_stamp', 'time_stamp'], axis=1) data_tdx.index = pd.to_datetime(data_tdx.index) res = pd.concat([res, data_tdx], sort=True) res = QA_DataStruct_Stock_day( res.reset_index().set_index(['datetime', 'code'])) except: res = None if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_stock_min_adv( code, start, end, frequence=frequence ) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_stock_min( code, start, end, frequence=frequence ) res = QA_DataStruct_Stock_min( res.set_index(['datetime', 'code']) ) elif market == MARKET_TYPE.FUTURE_CN: if frequence == FREQUENCE.DAY: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_future_day_adv(code, start, end) except: res = None if source == DATASOURCE.TDX or res is None: res = QATdx.QA_fetch_get_future_day(code, start, end) res = QA_DataStruct_Future_day(res.set_index(['date', 'code'])) elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_future_min_adv( code, start, end, frequence=frequence ) except: res = None if source == DATASOURCE.TDX or res is None: res = QATdx.QA_fetch_get_future_min( code, start, end, frequence=frequence ) res = QA_DataStruct_Future_min( res.set_index(['datetime', 'code']) ) elif market == MARKET_TYPE.INDEX_CN: if frequence == FREQUENCE.DAY: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_index_day_adv(code, start, end) except: return None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_index_day(code, start, end) res = QA_DataStruct_Index_day(res.set_index(['date', 'code'])) elif frequence in [FREQUENCE.ONE_MIN, FREQUENCE.FIVE_MIN, FREQUENCE.FIFTEEN_MIN, FREQUENCE.THIRTY_MIN, FREQUENCE.SIXTY_MIN]: if source == DATASOURCE.MONGO: try: res = QAQueryAdv.QA_fetch_index_min_adv( code, start, end, frequence=frequence ) except: res = None if source == DATASOURCE.TDX or res == None: res = QATdx.QA_fetch_get_index_min( code, start, end, frequence=frequence ) res = QA_DataStruct_Index_min( res.set_index(['datetime', 'code']) ) elif market == MARKET_TYPE.OPTION_CN: if source == DATASOURCE.MONGO: #res = QAQueryAdv.QA_fetch_option_day_adv(code, start, end) raise NotImplementedError('CURRENT NOT FINISH THIS METHOD') # print(type(res)) if output is OUTPUT_FORMAT.DATAFRAME: return res.data elif output is OUTPUT_FORMAT.DATASTRUCT: return res elif output is OUTPUT_FORMAT.NDARRAY: return res.to_numpy() elif output is OUTPUT_FORMAT.JSON: return res.to_json() elif output is OUTPUT_FORMAT.LIST: return res.to_list()
def QA_update_stock_day(name, startDate, endDate): data = QATushare.QA_fetch_get_stock_day(name, startDate, endDate)