def QA_fetch_financial_report(code, report_date, ltype='EN', db=DATABASE): """获取专业财务报表 Arguments: code {[type]} -- [description] report_date {[type]} -- [description] Keyword Arguments: ltype {str} -- [description] (default: {'EN'}) db {[type]} -- [description] (default: {DATABASE}) Raises: e -- [description] Returns: pd.DataFrame -- [description] """ if isinstance(code, str): code = [code] if isinstance(report_date, str): report_date = [QA_util_date_str2int(report_date)] elif isinstance(report_date, int): report_date = [report_date] elif isinstance(report_date, list): report_date = [QA_util_date_str2int(item) for item in report_date] collection = db.financial CH_columns = [item[3:] for item in list(financial_dict.keys())] CH_columns.extend(['277', '278', '279', '280', '281', '282', '_id', 'code', 'report_date']) CH_columns = pd.Index(CH_columns) EN_columns = list(financial_dict.values()) EN_columns.extend(['277', '278', '279', '280', '281', '282', '_id', 'code', 'report_date']) EN_columns = pd.Index(EN_columns) try: if code is not None and report_date is not None: data = [item for item in collection.find( {'code': {'$in': code}, 'report_date': {'$in': report_date}})] elif code is None and report_date is not None: data = [item for item in collection.find( {'report_date': {'$in': report_date}})] elif code is not None and report_date is None: data = [item for item in collection.find({'code': {'$in': code}})] else: data = [item for item in collection.find()] if len(data) > 0: res_pd = pd.DataFrame(data) if ltype in ['CH', 'CN']: res_pd.columns = CH_columns elif ltype is 'EN': res_pd.columns = EN_columns return res_pd.replace(-4.039810335e+34, numpy.nan).set_index(['report_date', 'code'], drop=False) else: return None except Exception as e: raise e
def __QA_fetch_get_stock_transaction(code, day, retry, api): data_ = pd.concat([api.to_df(api.get_history_transaction_data( __select_market_code(str(code)), str(code), (20 - i) * 800, 800, QA_util_date_str2int(day))) for i in range(21)], axis=0) for _ in range(retry): if len(data_) < 2: return __QA_fetch_get_stock_transaction(code, day, 0, api) else: return data_.assign(date=day).assign(datetime=pd.to_datetime(data_['time'].apply(lambda x: str(day) + ' ' + x)))\ .assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def __QA_fetch_get_stock_transaction(code, day, retry, api): market_code = __select_market_code(code) data_ = [] for i in range(21): data_ += api.get_history_transaction_data(market_code, code, (20 - i) * 800, 800, QA_util_date_str2int(day)) data_ = api.to_df(data_) data_['date'] = day data_['datetime'] = data_['time'].apply(lambda x: str(day) + ' ' + x) data_['datetime'] = pd.to_datetime(data_['datetime']) data_['code'] = str(code) data_['order'] = range(len(data_.index)) data_ = data_.set_index('datetime', drop=True) for _ in range(retry): if len(data_) < 2: return __QA_fetch_get_stock_transaction(code, day, 0, api) else: return data_
def __QA_fetch_get_stock_transaction(code, day, retry, api): batch_size = 2000 # 800 or 2000 ? 2000 maybe also works data_arr = [] max_offset = 21 cur_offset = 0 while cur_offset <= max_offset: one_chunk = api.get_history_transaction_data( _select_market_code(str(code)), str(code), cur_offset * batch_size, batch_size, QA_util_date_str2int(day)) if one_chunk is None or one_chunk == []: break data_arr = one_chunk + data_arr cur_offset += 1 data_ = api.to_df(data_arr) for _ in range(retry): if len(data_) < 2: return __QA_fetch_get_stock_transaction(code, day, 0, api) else: return data_.assign(date=day).assign(datetime=pd.to_datetime(data_['time'].apply(lambda x: str(day) + ' ' + x)))\ .assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def __QA_fetch_get_future_transaction(code, day, retry, code_market, apix): batch_size = 1800 # 800 or 2000 ? 2000 maybe also works data_arr = [] max_offset = 40 cur_offset = 0 while cur_offset <= max_offset: one_chunk = apix.get_history_transaction_data( code_market, str(code), QA_util_date_str2int(day), cur_offset * batch_size) if one_chunk is None or one_chunk == []: break data_arr = one_chunk + data_arr cur_offset += 1 data_ = apix.to_df(data_arr) for _ in range(retry): if len(data_) < 2: return __QA_fetch_get_stock_transaction(code, day, 0, apix) else: return data_.assign(datetime=pd.to_datetime(data_['date'])).assign(date=str(day))\ .assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def QA_fetch_financial_report(code, report_date, ltype='EN', db=DATABASE): """获取专业财务报表 Arguments: code {[type]} -- [description] report_date {[type]} -- [description] Keyword Arguments: ltype {str} -- [description] (default: {'EN'}) db {[type]} -- [description] (default: {DATABASE}) Raises: e -- [description] Returns: pd.DataFrame -- [description] """ if isinstance(code, str): code = [code] if isinstance(report_date, str): report_date = [QA_util_date_str2int(report_date)] elif isinstance(report_date, int): report_date = [report_date] elif isinstance(report_date, list): report_date = [QA_util_date_str2int(item) for item in report_date] collection = db.financial num_columns = [item[:3] for item in list(financial_dict.keys())] CH_columns = [item[3:] for item in list(financial_dict.keys())] EN_columns = list(financial_dict.values()) #num_columns.extend(['283', '_id', 'code', 'report_date']) # CH_columns.extend(['283', '_id', 'code', 'report_date']) #CH_columns = pd.Index(CH_columns) #EN_columns = list(financial_dict.values()) #EN_columns.extend(['283', '_id', 'code', 'report_date']) #EN_columns = pd.Index(EN_columns) try: if code is not None and report_date is not None: data = [item for item in collection.find( {'code': {'$in': code}, 'report_date': {'$in': report_date}}, {"_id": 0}, batch_size=10000)] elif code is None and report_date is not None: data = [item for item in collection.find( {'report_date': {'$in': report_date}}, {"_id": 0}, batch_size=10000)] elif code is not None and report_date is None: data = [item for item in collection.find( {'code': {'$in': code}}, {"_id": 0}, batch_size=10000)] else: data = [item for item in collection.find({}, {"_id": 0})] if len(data) > 0: res_pd = pd.DataFrame(data) if ltype in ['CH', 'CN']: cndict = dict(zip(num_columns, CH_columns)) cndict['283'] = '283' try: cndict['284'] = '284' cndict['285'] = '285' cndict['286'] = '286' except: pass cndict['code'] = 'code' cndict['report_date'] = 'report_date' res_pd.columns = res_pd.columns.map(lambda x: cndict[x]) elif ltype is 'EN': endict = dict(zip(num_columns, EN_columns)) endict['283'] = '283' try: endict['284'] = '284' endict['285'] = '285' endict['286'] = '286' except: pass endict['code'] = 'code' endict['report_date'] = 'report_date' res_pd.columns = res_pd.columns.map(lambda x: endict[x]) if res_pd.report_date.dtype == numpy.int64: res_pd.report_date = pd.to_datetime( res_pd.report_date.apply(QA_util_date_int2str)) else: res_pd.report_date = pd.to_datetime(res_pd.report_date) return res_pd.replace(-4.039810335e+34, numpy.nan).set_index(['report_date', 'code'], drop=False) else: return None except Exception as e: raise e
def QA_fetch_get_future_min(code, start, end, frequence=FREQUENCE.ONE_MIN): ''' TODO 当前仅支持L8和L9 期货数据 分钟线 code = 'IL8' start = '2014-12-27' end = '2014-12-31' ''' market_type = MARKET_TYPE.FUTURE_CN try: if code[-2:] in ['L8']: if frequence == FREQUENCE.ONE_MIN: '''找到总路径''' product_type = '股指期货' if code[:-2] in ['IF', 'IC', 'IH' ] else '商品期货' data_type = '主连' if code[-2:] == 'L8' else '指数' path = cofund_data_path[(market_type, frequence, product_type, data_type)] main_code = code[:-2] '''确定品种路径''' if main_code.isupper(): file_code = main_code if main_code in QA_util_listdir( path) else main_code.lower() elif main_code.islower(): file_code = main_code if main_code in QA_util_listdir( path) else main_code.upper() else: raise NotImplementedError path = os.path.abspath(path + file_code) date_calendar = QA_util_get_trade_range( max( start[:10], QA_util_date_int2str(min(QA_util_listfile(path, 'csv')))), min((end[:10]), QA_util_date_int2str(max(QA_util_listfile(path, 'csv'))))) # date_calendar = QA_util_get_trade_range(start[:10],end[:10]) data = pd.DataFrame() for date in date_calendar: try: intdate = QA_util_date_str2int(date) file_temp = pd.read_csv( os.path.join(path, str(intdate) + '.csv')) data = data.append(file_temp) except: pass # data = data[data['Time']>=85900] data['type'] = frequence data['code'] = code data = data.drop_duplicates(subset=['RealDate', 'Time']) return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)( DATASOURCE.COFUND, data)[start:end] else: print('当前仅支持1分钟的数据调用') raise NotImplementedError else: print('当前仅支持L8的数据调用') raise NotImplementedError except Exception as exp: print("code is ", code) print(exp.__str__) return None