def QA_fetch_financial_report(code, report_date, ltype='EN', db=DATABASE):
"""获取专业财务报表
Arguments:
code {[type]} -- [description]
report_date {[type]} -- [description]
Keyword Arguments:
ltype {str} -- [description] (default: {'EN'})
db {[type]} -- [description] (default: {DATABASE})
Raises:
e -- [description]
Returns:
pd.DataFrame -- [description]
"""
if isinstance(code, str):
code = [code]
if isinstance(report_date, str):
report_date = [QA_util_date_str2int(report_date)]
elif isinstance(report_date, int):
report_date = [report_date]
elif isinstance(report_date, list):
report_date = [QA_util_date_str2int(item) for item in report_date]
collection = db.financial
CH_columns = [item[3:] for item in list(financial_dict.keys())]
CH_columns.extend(['277', '278', '279', '280', '281', '282', '_id', 'code',
'report_date'])
CH_columns = pd.Index(CH_columns)
EN_columns = list(financial_dict.values())
EN_columns.extend(['277', '278', '279', '280', '281', '282', '_id', 'code',
'report_date'])
EN_columns = pd.Index(EN_columns)
try:
if code is not None and report_date is not None:
data = [item for item in collection.find(
{'code': {'$in': code}, 'report_date': {'$in': report_date}})]
elif code is None and report_date is not None:
data = [item for item in collection.find(
{'report_date': {'$in': report_date}})]
elif code is not None and report_date is None:
data = [item for item in collection.find({'code': {'$in': code}})]
else:
data = [item for item in collection.find()]
if len(data) > 0:
res_pd = pd.DataFrame(data)
if ltype in ['CH', 'CN']:
res_pd.columns = CH_columns
elif ltype is 'EN':
res_pd.columns = EN_columns
return res_pd.replace(-4.039810335e+34, numpy.nan).set_index(['report_date', 'code'], drop=False)
else:
return None
except Exception as e:
raise e
def __QA_fetch_get_stock_transaction(code, day, retry, api):
data_ = pd.concat([api.to_df(api.get_history_transaction_data(
__select_market_code(str(code)), str(code), (20 - i) * 800, 800, QA_util_date_str2int(day))) for i in range(21)], axis=0)
for _ in range(retry):
if len(data_) < 2:
return __QA_fetch_get_stock_transaction(code, day, 0, api)
else:
return data_.assign(date=day).assign(datetime=pd.to_datetime(data_['time'].apply(lambda x: str(day) + ' ' + x)))\
.assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def __QA_fetch_get_stock_transaction(code, day, retry, api):
market_code = __select_market_code(code)
data_ = []
for i in range(21):
data_ += api.get_history_transaction_data(market_code, code,
(20 - i) * 800, 800,
QA_util_date_str2int(day))
data_ = api.to_df(data_)
data_['date'] = day
data_['datetime'] = data_['time'].apply(lambda x: str(day) + ' ' + x)
data_['datetime'] = pd.to_datetime(data_['datetime'])
data_['code'] = str(code)
data_['order'] = range(len(data_.index))
data_ = data_.set_index('datetime', drop=True)
for _ in range(retry):
if len(data_) < 2:
return __QA_fetch_get_stock_transaction(code, day, 0, api)
else:
return data_
def __QA_fetch_get_stock_transaction(code, day, retry, api):
batch_size = 2000 # 800 or 2000 ? 2000 maybe also works
data_arr = []
max_offset = 21
cur_offset = 0
while cur_offset <= max_offset:
one_chunk = api.get_history_transaction_data(
_select_market_code(str(code)), str(code), cur_offset * batch_size,
batch_size, QA_util_date_str2int(day))
if one_chunk is None or one_chunk == []:
break
data_arr = one_chunk + data_arr
cur_offset += 1
data_ = api.to_df(data_arr)
for _ in range(retry):
if len(data_) < 2:
return __QA_fetch_get_stock_transaction(code, day, 0, api)
else:
return data_.assign(date=day).assign(datetime=pd.to_datetime(data_['time'].apply(lambda x: str(day) + ' ' + x)))\
.assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def __QA_fetch_get_future_transaction(code, day, retry, code_market, apix):
batch_size = 1800 # 800 or 2000 ? 2000 maybe also works
data_arr = []
max_offset = 40
cur_offset = 0
while cur_offset <= max_offset:
one_chunk = apix.get_history_transaction_data(
code_market, str(code), QA_util_date_str2int(day), cur_offset * batch_size)
if one_chunk is None or one_chunk == []:
break
data_arr = one_chunk + data_arr
cur_offset += 1
data_ = apix.to_df(data_arr)
for _ in range(retry):
if len(data_) < 2:
return __QA_fetch_get_stock_transaction(code, day, 0, apix)
else:
return data_.assign(datetime=pd.to_datetime(data_['date'])).assign(date=str(day))\
.assign(code=str(code)).assign(order=range(len(data_.index))).set_index('datetime', drop=False, inplace=False)
def QA_fetch_financial_report(code, report_date, ltype='EN', db=DATABASE):
"""获取专业财务报表
Arguments:
code {[type]} -- [description]
report_date {[type]} -- [description]
Keyword Arguments:
ltype {str} -- [description] (default: {'EN'})
db {[type]} -- [description] (default: {DATABASE})
Raises:
e -- [description]
Returns:
pd.DataFrame -- [description]
"""
if isinstance(code, str):
code = [code]
if isinstance(report_date, str):
report_date = [QA_util_date_str2int(report_date)]
elif isinstance(report_date, int):
report_date = [report_date]
elif isinstance(report_date, list):
report_date = [QA_util_date_str2int(item) for item in report_date]
collection = db.financial
num_columns = [item[:3] for item in list(financial_dict.keys())]
CH_columns = [item[3:] for item in list(financial_dict.keys())]
EN_columns = list(financial_dict.values())
#num_columns.extend(['283', '_id', 'code', 'report_date'])
# CH_columns.extend(['283', '_id', 'code', 'report_date'])
#CH_columns = pd.Index(CH_columns)
#EN_columns = list(financial_dict.values())
#EN_columns.extend(['283', '_id', 'code', 'report_date'])
#EN_columns = pd.Index(EN_columns)
try:
if code is not None and report_date is not None:
data = [item for item in collection.find(
{'code': {'$in': code}, 'report_date': {'$in': report_date}}, {"_id": 0}, batch_size=10000)]
elif code is None and report_date is not None:
data = [item for item in collection.find(
{'report_date': {'$in': report_date}}, {"_id": 0}, batch_size=10000)]
elif code is not None and report_date is None:
data = [item for item in collection.find(
{'code': {'$in': code}}, {"_id": 0}, batch_size=10000)]
else:
data = [item for item in collection.find({}, {"_id": 0})]
if len(data) > 0:
res_pd = pd.DataFrame(data)
if ltype in ['CH', 'CN']:
cndict = dict(zip(num_columns, CH_columns))
cndict['283'] = '283'
try:
cndict['284'] = '284'
cndict['285'] = '285'
cndict['286'] = '286'
except:
pass
cndict['code'] = 'code'
cndict['report_date'] = 'report_date'
res_pd.columns = res_pd.columns.map(lambda x: cndict[x])
elif ltype is 'EN':
endict = dict(zip(num_columns, EN_columns))
endict['283'] = '283'
try:
endict['284'] = '284'
endict['285'] = '285'
endict['286'] = '286'
except:
pass
endict['code'] = 'code'
endict['report_date'] = 'report_date'
res_pd.columns = res_pd.columns.map(lambda x: endict[x])
if res_pd.report_date.dtype == numpy.int64:
res_pd.report_date = pd.to_datetime(
res_pd.report_date.apply(QA_util_date_int2str))
else:
res_pd.report_date = pd.to_datetime(res_pd.report_date)
return res_pd.replace(-4.039810335e+34, numpy.nan).set_index(['report_date', 'code'], drop=False)
else:
return None
except Exception as e:
raise e
def QA_fetch_get_future_min(code, start, end, frequence=FREQUENCE.ONE_MIN):
'''
TODO 当前仅支持L8和L9
期货数据 分钟线
code = 'IL8'
start = '2014-12-27'
end = '2014-12-31'
'''
market_type = MARKET_TYPE.FUTURE_CN
try:
if code[-2:] in ['L8']:
if frequence == FREQUENCE.ONE_MIN:
'''找到总路径'''
product_type = '股指期货' if code[:-2] in ['IF', 'IC', 'IH'
] else '商品期货'
data_type = '主连' if code[-2:] == 'L8' else '指数'
path = cofund_data_path[(market_type, frequence, product_type,
data_type)]
main_code = code[:-2]
'''确定品种路径'''
if main_code.isupper():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.lower()
elif main_code.islower():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.upper()
else:
raise NotImplementedError
path = os.path.abspath(path + file_code)
date_calendar = QA_util_get_trade_range(
max(
start[:10],
QA_util_date_int2str(min(QA_util_listfile(path,
'csv')))),
min((end[:10]),
QA_util_date_int2str(max(QA_util_listfile(path,
'csv')))))
# date_calendar = QA_util_get_trade_range(start[:10],end[:10])
data = pd.DataFrame()
for date in date_calendar:
try:
intdate = QA_util_date_str2int(date)
file_temp = pd.read_csv(
os.path.join(path,
str(intdate) + '.csv'))
data = data.append(file_temp)
except:
pass
# data = data[data['Time']>=85900]
data['type'] = frequence
data['code'] = code
data = data.drop_duplicates(subset=['RealDate', 'Time'])
return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)(
DATASOURCE.COFUND, data)[start:end]
else:
print('当前仅支持1分钟的数据调用')
raise NotImplementedError
else:
print('当前仅支持L8的数据调用')
raise NotImplementedError
except Exception as exp:
print("code is ", code)
print(exp.__str__)
return None