def QA_SU_save_deal(dealist, client=DATABASE): """存储order_handler的deal_status Arguments: dealist {[dataframe]} -- [description] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ if isinstance(dealist, pd.DataFrame): collection = client.deal collection.create_index( [('account_cookie', ASCENDING), ('trade_id', ASCENDING)], unique=True ) try: dealist = QA_util_to_json_from_pandas(dealist.reset_index()) collection.insert_many(dealist, ordered=False) except Exception as e: pass
def __init__(self, market_data): """转化成dict模式 Arguments: market_data {[type]} -- [description] """ if isinstance(market_data, dict): self.market_data = market_data elif isinstance(market_data, pd.DataFrame): self.market_data = QA_util_to_json_from_pandas(market_data)
def __saving_work(code): QA_util_log_info('##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format( str(code)), ui_log=ui_log) try: coll.insert_many( QA_util_to_json_from_pandas( # 🛠todo str(stock_list[code]) 参数不对? QA_fetch_get_stock_transaction(str(code), str(stock_list[code]), str(now_time())[0:10]))) except: err.append(str(code))
def __saving_work(code, coll_stock_year): try: QA_util_log_info('##JOB01 Now Saving STOCK_YEAR==== {}'.format( str(code))) ref = coll_stock_year.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: coll_stock_year.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), QA_util_get_next_day(start_date), end_date, '00', frequence='year'))) else: start_date = '1990-01-01' QA_util_log_info( 'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: coll_stock_year.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00', frequence='year'))) except: err.append(str(code))
def __saving_work(code, stock_shares): try: QA_util_log_info( '##JOB01 Now Saving SSINA shares change==== {}'.format( str(code)), ui_log) stock_shares.insert_many(QA_util_to_json_from_pandas( QA_fetch_get_stock_shares_sina(code)), ordered=False) gc.collect() except Exception as error0: print(error0) err.append(str(code))
def __saving_work(code, stock_financial_sina): try: QA_util_log_info( '##JOB01 Now Saving SINA financial_report==== {}'.format( str(code)), ui_log) stock_financial_sina.insert_many(QA_util_to_json_from_pandas( QA_fetch_get_stock_report_sina(code, YEARS)), ordered=False) gc.collect() except Exception as error0: print(error0) err.append(str(code))
def QA_SU_save_deal(dealist, client=DATABASE): """存储order_handler的deal_status Arguments: dealist {[dataframe]} -- [description] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ if isinstance(dealist, pd.DataFrame): dealist = QA_util_to_json_from_pandas(dealist.reset_index()) collection = client.deal collection.create_index([('account_cookie', ASCENDING), ('trade_id', ASCENDING)], unique=True) try: collection.insert_many(dealist, ordered=False) except Exception as e: pass
def QA_fetch_stock_fianacial(code, start, end=None, format='pd', collections=DATABASE.stock_financial_analysis): '获取quant基础数据' #code= [code] if isinstance(code,str) else code # code checking code = QA_util_code_tolist(code) if QA_util_date_valid(end): cursor = collections.find( { 'CODE': { '$in': code }, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start) } }, batch_size=10000) #res=[QA_util_dict_remove_key(data, '_id') for data in cursor] res = pd.DataFrame([item for item in cursor]) try: res.columns = [ i.lower() if i == 'CODE' else i for i in list(res.columns) ] res = res.drop(['date_stamp', '_id'], axis=1).drop_duplicates( (['code', 'date'])) res['RNG_RES'] = res['AVG60_RNG'] * 60 / res['RNG_60'] except: res = None if format in ['P', 'p', 'pandas', 'pd']: #res['report_date'] = pd.to_datetime(res['report_date']/1000, unit='s') return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "QA Error QA_fetch_financial_TTM format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None else: QA_util_log_info( 'QA Error QA_fetch_financial_TTM data parameter start=%s end=%s is not right' % (start, end))
def QA_fetch_index_day(code, start, end, format='numpy', collections=DATABASE.index_day): '获取指数日线' start = str(start)[0:10] end = str(end)[0:10] code = QA_util_code_tolist(code) if QA_util_date_valid(end) == True: cursor = collections.find( { 'code': { '$in': code }, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start) } }, {"_id": 0}, batch_size=10000) res = pd.DataFrame([item for item in cursor]) try: res = res.assign(volume=res.vol, date=pd.to_datetime(res.date)).drop_duplicates( (['date', 'code' ])).query('volume>1').set_index('date', drop=False) except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "QA Error QA_fetch_index_day format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" " % format) return None else: QA_util_log_info( 'QA Error QA_fetch_index_day data parameter start=%s end=%s is not right' % (start, end))
def QA_ts_update_namechange(): """ 保存所有股票的历史曾用名 """ # 建表 coll = DATABASE.namechange coll.create_index( [ ("code", ASCENDING), ("start_date_stamp", ASCENDING), ("end_date_stamp", ASCENDING), ("ann_date_stamp", ASCENDING), ], unique=True, ) # 初始化数据接口 pro = get_pro() # 获取历史所有股票 symbol_list = sorted( list(set(QA_fmt_code_list(QA_fetch_stock_basic().index.tolist(), "ts")))) df = pd.DataFrame() for i, symbol in enumerate(symbol_list): if i % 100 == 0: print(f"Saving {i}th stock name, stock is {symbol}") try: df = df.append(pro.namechange(ts_code=symbol)) except: time.sleep(1) try: df = df.append(pro.namechange(ts_code=symbol)) except: raise ValueError("[ERROR]\t数据获取失败") # df.to_csv("test.csv") df["code"] = QA_fmt_code_list(df["ts_code"]) df["start_date_stamp"] = df["start_date"].apply(QA_util_date_stamp) df["end_date_stamp"] = df["end_date"].apply(QA_util_date_stamp) df["ann_date_stamp"] = df["ann_date"].apply(QA_util_date_stamp) df = df.where(pd.notnull(df), None) js = QA_util_to_json_from_pandas(df.drop(columns=["ts_code"])) for item in js: if not item["end_date"]: item["end_date_stamp"] = 9999999999 qry = { "code": item["code"], "start_date_stamp": item["start_date_stamp"], "end_date_stamp": item["end_date_stamp"], "ann_date_stamp": item["ann_date_stamp"], } if coll.count_documents(qry) == 0: coll.insert_one(item)
def __saving_work(code, coll): QA_util_log_info('##JOB07 Now Saving ETF_MIN ==== {}'.format( str(code))) try: for type in ['1min', '5min', '15min', '30min', '60min']: ref_ = coll.find({'code': str(code)[0:6], 'type': type}) end_time = str(now_time())[0:19] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['datetime'] QA_util_log_info( '##JOB07.{} Now Saving {} from {} to {} =={} '.format( ['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_index_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data[1::])) else: start_time = '2015-01-01' QA_util_log_info( '##JOB07.{} Now Saving {} from {} to {} =={} '.format( ['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_index_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data)) except: err.append(code)
def QA_fetch_stock_alpha(code, start, end=None, format='pd', collections=DATABASE.stock_alpha): '获取股票日线' #code= [code] if isinstance(code,str) else code # code checking code = QA_util_code_tolist(code) if QA_util_date_valid(end): __data = [] cursor = collections.find( { 'code': { '$in': code }, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start) } }, {"_id": 0}, batch_size=10000) #res=[QA_util_dict_remove_key(data, '_id') for data in cursor] res = pd.DataFrame([item for item in cursor]) try: res = res.drop_duplicates((['code', 'date'])).set_index(['date', 'code']) except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "QA Error QA_fetch_stock_alpha format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None else: QA_util_log_info( 'QA Error QA_fetch_stock_alpha data parameter start=%s end=%s is not right' % (start, end))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_hfq = QA_fetch_get_stock_day( i, start='1990-01-01', if_fq='02', type_='pd' ) data_json = QA_util_to_json_from_pandas(data_hfq) __coll.insert_many(data_json) except Exception as e: print(e) QA_util_log_info('error in saving ==== %s' % str(i))
def get(self): action = self.get_argument('action', 'acchistory') if action == 'accountlist': res = self.manager.get_allaccountname() self.write({'res': res}) elif action == 'portfoliolist': res = self.manager.get_allportfolio() self.write({'res': res}) elif action == 'accountinportfolio': portfolio = self.get_argument('portfolio', 't12') res = self.manager.get_portfolio_panel(portfolio) self.write({'res': QA_util_to_json_from_pandas(res)})
def QA_fetch_get_trade_date(end, exchange): data = QATs.trade_cal() da = data[data.isOpen > 0] data_json = QA_util_to_json_from_pandas(data) message = [] for i in range(0, len(data_json) - 1, 1): date = data_json[i]['calendarDate'] num = i + 1 exchangeName = 'SSE' data_stamp = QA_util_date_stamp(date) mes = {'date': date, 'num': num, 'exchangeName': exchangeName, 'date_stamp': data_stamp} message.append(mes) return message
def __saving_work(code, coll): QA_util_log_info( '##JOB02 Now Saving XDXR INFO ==== {}'.format(str(code)), ui_log=ui_log ) try: coll.insert_many( QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr(str(code))), ordered=False ) except: err.append(str(code))
def _ts_update_all(report_date, report_type, sheet_type, wait_seconds, max_trial): # 获取指定报告期,指定报告类型,指定报表类型的截面数据 df = QA_fetch_get_crosssection_financial( report_date=report_date, report_type=report_type, sheet_type=sheet_type, wait_seconds=wait_seconds, max_trial=max_trial, ) coll = eval(f"DATABASE.{sheet_type}") # 考虑到查找的方式,可能根据股票代码查找,可能根据报告期查找,可能根据公告期查找,可能根据最后公告期查找,可能根据报告类型查找 coll.create_index( [ ("code", ASCENDING), ("report_label", ASCENDING), ("report_date", DESCENDING), ("report_type", ASCENDING), ("report_date_stamp", ASCENDING), ("ann_date_stamp", ASCENDING), ("f_ann_date_stamp", ASCENDING), ], unique=False, ) # FIXME: insert_many may be better for item in QA_util_to_json_from_pandas(df): report_label = None if item["report_date"].endswith("0331"): report_label = "1" elif item["report_date"].endswith("0630"): report_label = "2" elif item["report_date"].endswith("0930"): report_label = "3" elif item["report_date"].endswith("1231"): report_label = "4" item["report_label"] = report_label coll.update_one( { "code": item["code"], "report_label": report_label, "report_date": item["report_date"], "report_type": item["report_type"], "report_date_stamp": QA_util_date_stamp( item["report_date"]), "ann_date_stamp": QA_util_date_stamp(item["ann_date"]), "f_ann_date_stamp": QA_util_date_stamp(item["f_ann_date"]), }, {"$set": item}, upsert=True, )
def QA_SU_save_order(orderlist, client=DATABASE): """存储order_handler的order_status Arguments: orderlist {[dataframe]} -- [description] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ orderlist = QA_util_to_json_from_pandas(orderlist.reset_index()) collection = client.order collection.create_index( [('account_cookie', ASCENDING), ('realorder_id', ASCENDING)], unique=True) try: for item in orderlist: if item: #item['date']= QA_util_get_order_day() collection.update_one({'account_cookie': item.get('account_cookie'), 'realorder_id': item.get('realorder_id')}, {'$set': item}, upsert=True) except Exception as e: print(e) pass
def QA_SU_save_order(orderlist, client=DATABASE): """存储order_handler的order_status Arguments: orderlist {[dataframe]} -- [description] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ if isinstance(orderlist, pd.DataFrame): collection = client.order collection.create_index( [('account_cookie', ASCENDING), ('realorder_id', ASCENDING)], unique=True ) try: orderlist = QA_util_to_json_from_pandas(orderlist.reset_index()) for item in orderlist: if item: #item['date']= QA_util_get_order_day() collection.update_one( { 'account_cookie': item.get('account_cookie'), 'realorder_id': item.get('realorder_id') }, {'$set': item}, upsert=True ) except Exception as e: print(e) pass
async def QA_fetch_stock_min(code, start, end, format='numpy', frequence='1min', collections=DATABASE_ASYNC.stock_min): '获取股票分钟线' if frequence in ['1min', '1m']: frequence = '1min' elif frequence in ['5min', '5m']: frequence = '5min' elif frequence in ['15min', '15m']: frequence = '15min' elif frequence in ['30min', '30m']: frequence = '30min' elif frequence in ['60min', '60m']: frequence = '60min' else: print("QA Error QA_fetch_stock_min parameter frequence=%s is none of 1min 1m 5min 5m 15min 15m 30min 30m 60min 60m" % frequence) __data = [] # code checking code = QA_util_code_tolist(code) cursor = collections.find({ 'code': {'$in': code}, "time_stamp": { "$gte": QA_util_time_stamp(start), "$lte": QA_util_time_stamp(end) }, 'type': frequence }) try: res = pd.DataFrame([item async for item in cursor]) except SyntaxError: print('THIS PYTHON VERSION NOT SUPPORT "async for" function') pass try: res = res.drop('_id', axis=1).assign(volume=res.vol).query('volume>1').assign(datetime=pd.to_datetime( res.datetime)).drop_duplicates(['datetime', 'code']).set_index('datetime', drop=False) # return res except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print("QA Error QA_fetch_stock_min format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None
def QA_fetch_stock_day(code, start, end, format='numpy', frequence='day', collections=DATABASE.stock_day): """'获取股票日线' Returns: [type] -- [description] 感谢@几何大佬的提示 https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-the-specified-fields-and-the-id-field-only """ start = str(start)[0:10] end = str(end)[0:10] #code= [code] if isinstance(code,str) else code # code checking code = QA_util_code_tolist(code) if QA_util_date_valid(end): cursor = collections.find({ 'code': {'$in': code}, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start)}}, {"_id": 0}, batch_size=10000) #res=[QA_util_dict_remove_key(data, '_id') for data in cursor] res = pd.DataFrame([item for item in cursor]) try: res = res.assign(volume=res.vol, date=pd.to_datetime( res.date)).drop_duplicates((['date', 'code'])).query('volume>1').set_index('date', drop=False) res = res.ix[:, ['code', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date']] except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print("QA Error QA_fetch_stock_day format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None else: QA_util_log_info( 'QA Error QA_fetch_stock_day data parameter start=%s end=%s is not right' % (start, end))
def __saving_work(code, coll): QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code))) try: for type in ['1min', '5min', '15min', '30min', '60min']: ref_ = coll.find({'code': str(code)[0:6], 'type': type}) end_time = str(now_time())[0:19] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['datetime'] QA_util_log_info( '##JOB03.%s Now Saving %s from %s to %s ==%s ' % ([ '1min', '5min', '15min', '30min', '60min' ].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_stock_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data[1::])) else: start_time = '2015-01-01' QA_util_log_info( '##JOB03.%s Now Saving %s from %s to %s ==%s ' % ([ '1min', '5min', '15min', '30min', '60min' ].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_stock_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data)) except Exception as e: QA_util_log_info(e) err.append(code)
def QA_fetch_interest_rate(start, end=None, format='pd', collections=DATABASE.interest_rate): '获取股票日线' #code= [code] if isinstance(code,str) else code # code checking if end is None: end = QA_util_today_str() if start is None: start = '1999-01-01' if QA_util_date_valid(end): __data = [] cursor = collections.find( { "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start) } }, {"_id": 0}, batch_size=10000) #res=[QA_util_dict_remove_key(data, '_id') for data in cursor] res = pd.DataFrame([item for item in cursor]) try: res = res.drop(columns=['crawl_date', 'date_stamp']) except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "QA Error Interest Rate format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None else: QA_util_log_info( 'QA Error Interest Rate data parameter start=%s end=%s is not right' % (start, end))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_bfq = QATushare.QA_fetch_get_stock_day(i, startDate='1990-01-01', if_fq='00', type_='pd') data_qfq = QATushare.QA_fetch_get_stock_day(i, startDate='1990-01-01', if_fq='01', type_='pd') data_qfq['qfqfactor'] = data_qfq['open'] / data_bfq['open'] data_json = QA_util_to_json_from_pandas(data_qfq) __coll.insert_many(data_json) except: QA_util_log_info('error in saving ==== %s' % str(i))
def QA_SU_save_daily_basic(client=DATABASE, date=None): __coll = client.daily_basic_tushare ''' __coll.create_index( [("ts_code", pymongo.ASCENDING), ("trade_date", pymongo.ASCENDING)] )''' if (date == None): # date = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y%m%d') date = trading_calendar_utils.get_last_x_trading_day_from_mongodb() date = str(date) data = QA_fetch_get_stock_daily_basic(date) if (data is not None): __coll.insert_many(QA_util_to_json_from_pandas(data))
def fetch_index_day_common(field_name, field_value, start, end,collections, format='pd'): '通用数据查询' start = str(start)[0:10] end = str(end)[0:10] field_value_ = field_value if not isinstance(field_value_, list): field_value_ = [field_value_] cursor = collections.find( { field_name: { '$in': field_value_ }, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start) } }, {"_id": 0}, batch_size=10000 ) res = pd.DataFrame([item for item in cursor]) try: res = res.assign( date=pd.to_datetime(res.date, utc=False) ).set_index('date',drop=True) except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "Error fetch_index_day_em format parameter %s is none of \"P, p, pandas, pd , n, N, numpy !\" " % format ) return None
def QA_save_fq_factor(client=QA_Setting.client): '保存复权因子' _coll = client.quantaxis.fq_factor _stock_list = QA_fetch_get_stock_list() for _item in _stock_list: data = QA_data_fq_factor(_item) data['date'] = data.index data['date'] = data['date'].apply(lambda x: str(x)[0:10]) data['date_stamp'] = data['date'].apply( lambda x: QA_util_date_stamp(x)) data['code'] = _item data_json = QA_util_to_json_from_pandas(data) _coll.insert_many(data_json)
def QA_SU_save_stock_block(client=DATABASE, ui_log=None, ui_progress=None): """ Tushare的版块数据 Returns: [type] -- [description] """ coll = client.stock_block coll.create_index('code') try: # 暂时先只有中证500 csindex500 = QA_fetch_get_stock_block() coll.insert_many(QA_util_to_json_from_pandas(csindex500)) QA_util_log_info('SUCCESS save stock block ^_^', ui_log) except Exception as e: QA_util_log_info('ERROR CODE: ', ui_log) QA_util_log_info(e, ui_log)
def getMin(cls, code, start, end, if_fq='00', frequence=8): collections = cls.collectionsMin '获取股票分钟线' _, type_, _ = cls.getReverseFrequence(frequence) _data = [] # code checking code = QA_util_code_tolist(code) cursor = collections.find( { 'code': { '$in': code }, "time_stamp": { "$gte": QA_util_time_stamp(start), "$lte": QA_util_time_stamp(end) }, 'type': type_ }, {"_id": 0}, batch_size=10000) res = pd.DataFrame([item for item in cursor]) try: res = res.assign( volume=res.vol, datetime=pd.to_datetime( res.datetime)).query('volume>1').drop_duplicates( ['datetime', 'code']).set_index('datetime', drop=False) # return res except: res = None if cls.format in ['P', 'p', 'pandas', 'pd']: return res elif cls.format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif cls.format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif cls.format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print( "QA Error QA_fetch_stock_min format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % cls.format) return None
def QA_fetch_tech_indicator(code, start, end, keys='all', format='pd', collections=DATABASE.df_tech_indicator): """ return: from start to end, doesn't exclude vol==0 dates in between, those date are continuous and aligned with sh index already """ start = str(start)[0:10] end = str(end)[0:10] # code checking code = QA_util_code_tolist(code) if QA_util_date_valid(end): cursor = collections.find({ 'code': {'$in': code}, "date_stamp": { "$lte": QA_util_date_stamp(end), "$gte": QA_util_date_stamp(start)}}) res = pd.DataFrame([item for item in cursor]) try: # drop '_id', change 'date' from str to datetime, set 'date' to index res = res.drop('_id', axis=1).assign(date=pd.to_datetime( res.date)).drop_duplicates((['date', 'code'])).set_index('date', drop=False) if keys != 'all': if isinstance(keys, str): res = res.ix[:, [keys]] elif isinstance(keys, list): res = res.ix[:, keys] except: res = None if format in ['P', 'p', 'pandas', 'pd']: return res elif format in ['json', 'dict']: return QA_util_to_json_from_pandas(res) # 多种数据格式 elif format in ['n', 'N', 'numpy']: return numpy.asarray(res) elif format in ['list', 'l', 'L']: return numpy.asarray(res).tolist() else: print("QA Error QA_fetch_tech_indicator format parameter %s is none of \"P, p, pandas, pd , json, dict , n, N, numpy, list, l, L, !\" " % format) return None else: QA_util_log_info( 'QA Error QA_fetch_tech_indicator data parameter start=%s end=%s is not right' % (start, end))
def QA_SU_save_stock_fianacial_momgo(start_date=None,end_date=None, ui_log = None, ui_progress = None): if start_date is None: if end_date is None: start_date = QA_util_today_str() end_date = start_date elif end_date is not None: start_date = '2008-01-01' elif start_date is not None: if end_date == None: end_date = QA_util_today_str() elif end_date is not None: if end_date < start_date: print('end_date should large than start_date') col = DATABASE.stock_financial_analysis col.create_index( [("CODE", ASCENDING), ("date_stamp", ASCENDING)], unique=True) deal_date_list = list(pd.date_range(start_date, end_date).map(lambda t:str(t.date()))) if deal_date_list is None: print('not a trading day') else: for deal_date in deal_date_list: data = QA_util_etl_stock_quant(deal_date) if data is not None: data = data.drop_duplicates( (['CODE', 'date'])) QA_util_log_info( '##JOB01 Pre Data stock financial data ============== {deal_date} '.format(deal_date=deal_date), ui_log) data = QA_util_to_json_from_pandas(data) print("got stock financial data ============== {deal_date}".format(deal_date=deal_date)) QA_util_log_info( '##JOB02 Got Data stock financial data ============== {deal_date}'.format(deal_date=deal_date), ui_log) try: col.insert_many(data, ordered=False) QA_util_log_info( '##JOB03 Now stock financial data saved ============== {deal_date} '.format(deal_date=deal_date), ui_log) except Exception as e: if isinstance(e, MemoryError): col.insert_many(data, ordered=True) elif isinstance(e, pymongo.bulk.BulkWriteError): pass pass else: QA_util_log_info( '##JOB01 No Data stock_fianacial_data ============== {deal_date} '.format(deal_date=deal_date), ui_log)
def __saving_work(self, df=pd.DataFrame()): try: if not (df is None) and len(df) > 0: coll_stock_day = self.client.stock_day coll_stock_day.create_index([("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING) ]) coll_stock_day.insert_many(QA_util_to_json_from_pandas(df)) QA_util_log_info( '##JOB02 Now Saved STOCK_DAY==== {}'.format( df.code.unique()[0]), self.ui_log) else: QA_util_log_info( '##JOB02 Saving STOCK_DAY==== {}'.format('skipped'), self.ui_log) except Exception as error0: print(error0, flush=True)
def QA_SU_save_stock_list(client=DATABASE): """save stock_list Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ client.drop_collection('stock_list') coll = client.stock_list coll.create_index('code') err = [] try: QA_util_log_info('##JOB08 Now Saving STOCK_LIST ====') coll.insert_many(QA_util_to_json_from_pandas( QA_fetch_get_stock_list())) except: pass
class _realtime_base(): """ realtime 基类 主要字段有: code/name time open/high/low 买卖报价队列:(不同的可能不一样 只提供list) ask_list[ask1_price/ask1_volume|ask2_price/ask2_volume|ask3_price/ask3_volume....] bid_list[bid1_price/bid1_volume|bid2_price/bid2_volume|bid3_price/bid3_volume....] """ def __init__(self, market_data): """转化成dict模式 Arguments: market_data {[type]} -- [description] """ if isinstance(market_data, dict): self.market_data = market_data elif isinstance(market_data, pd.DataFrame): self.market_data = QA_util_to_json_from_pandas(market_data) @property def open(self): return self.market_data.get('open', None) @property def price(self): return self.market_data.get('price', None) @property def datetime(self): return self.market_data.get('datetime', None) @property def high(self): return self.market_data.get('high', None) @property def low(self): return self.market_data.get('low', None) @property def code(self): return self.market_data.get('code', None) @property def last_close(self): return self.market_data.get('last_close', None) @property def cur_vol(self): return self.market_data.get('cur_vol', None) @property def bid1(self): return self.market_data.get('bid1', None) @property def bid_vol1(self): return self.market_data.get('bid_vol1', None) @property def bid2(self): return self.market_data.get('bid2', None) @property def bid_vol2(self): return self.market_data.get('bid_vol2', None) @property def bid3(self): return self.market_data.get('bid3', None) @property def bid_vol3(self): return self.market_data.get('bid_vol3', None) @property def bid4(self): return self.market_data.get('bid4', None) @property def bid_vol4(self): return self.market_data.get('bid_vol4', None) @property def bid5(self): return self.market_data.get('bid5', None) @property def bid_vol5(self): return self.market_data.get('bid_vol5', None) @property def ask1(self): return self.market_data.get('ask1', None) @property def ask_vol1(self): return self.market_data.get('ask_vol1', None) @property def ask2(self): return self.market_data.get('ask2', None) @property def ask_vol2(self): return self.market_data.get('ask_vol2', None) @property def ask3(self): return self.market_data.get('ask3', None) @property def ask_vol3(self): return self.market_data.get('ask_vol3', None) @property def ask4(self): return self.market_data.get('ask4', None) @property def ask_vol4(self): return self.market_data.get('ask_vol4', None) @property def ask5(self): return self.market_data.get('ask5', None) @property def ask_vol5(self): return self.market_data.get('ask_vol5', None)