def QA_DataAggrement_Stock_info(data):
data['updated_date'] = list(
map(lambda x: QA_util_date_int2str(x), data['updated_date']))
data['ipo_date'] = list(
map(lambda x: QA_util_date_int2str(x), data['ipo_date']))
data['source'] = DATASOURCE.TDX
return data
def QA_SU_save_financial_files():
"""本地存储financialdata
"""
download_financialzip()
coll = DATABASE.financial
coll.create_index([("code", ASCENDING), ("report_date", ASCENDING)],
unique=True)
for item in os.listdir(download_path):
if item[0:4] != 'gpcw':
print(
"file ", item,
" is not start with gpcw , seems not a financial file , ignore!"
)
continue
date = int(item.split('.')[0][-8:])
print('QUANTAXIS NOW SAVING {}'.format(date))
if coll.find({'report_date': date}).count() < 3600:
print(coll.find({'report_date': date}).count())
data = parse_filelist([item]).reset_index().drop_duplicates(
subset=['code', 'report_date']).sort_index()
data["crawl_date"] = QA_util_today_str()
data = data.assign(report_date=data['report_date'].apply(
lambda x: QA_util_date_stamp(QA_util_date_int2str(x))))
data = data.assign(crawl_date=data['crawl_date'].apply(
lambda x: QA_util_date_stamp(QA_util_date_int2str(x))))
data = QA_util_to_json_from_pandas(data)
try:
coll.insert_many(data, ordered=False)
except Exception as e:
if isinstance(e, MemoryError):
coll.insert_many(data, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
else:
print('ALL READY IN DATABASE')
print('SUCCESSFULLY SAVE/UPDATE FINANCIAL DATA')
def QA_fetch_financial_code(ndays=10):
START = str(QA_util_get_pre_trade_date(QA_util_today_str(), ndays))
code = list(
QA_fetch_stock_financial_calendar(
QA.QA_fetch_stock_list_adv().code.tolist(), start=START)['code'])
market_day = pd.DataFrame(
QA_fetch_stock_basic_info_tushare())[['code', 'timeToMarket']]
market_day['TM'] = market_day['timeToMarket'].apply(lambda x: str(
QA_util_add_months(QA_util_date_int2str(int(x)), 0)
if x > 0 else None)[0:10])
code = list(market_day[market_day['TM'] >= START]['code'].values) + code
return (code)
def QA_fetch_get_stock_time_to_market():
data = QATs.get_stock_basics()
return data[data['timeToMarket'] != 0]['timeToMarket'].apply(lambda x: QA_util_date_int2str(x))
def QA_fetch_get_future_min(code, start, end, frequence=FREQUENCE.ONE_MIN):
'''
TODO 当前仅支持L8和L9
期货数据 分钟线
code = 'IL8'
start = '2014-12-27'
end = '2014-12-31'
'''
market_type = MARKET_TYPE.FUTURE_CN
try:
if code[-2:] in ['L8']:
if frequence == FREQUENCE.ONE_MIN:
'''找到总路径'''
product_type = '股指期货' if code[:-2] in ['IF', 'IC', 'IH'
] else '商品期货'
data_type = '主连' if code[-2:] == 'L8' else '指数'
path = cofund_data_path[(market_type, frequence, product_type,
data_type)]
main_code = code[:-2]
'''确定品种路径'''
if main_code.isupper():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.lower()
elif main_code.islower():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.upper()
else:
raise NotImplementedError
path = os.path.abspath(path + file_code)
date_calendar = QA_util_get_trade_range(
max(
start[:10],
QA_util_date_int2str(min(QA_util_listfile(path,
'csv')))),
min((end[:10]),
QA_util_date_int2str(max(QA_util_listfile(path,
'csv')))))
# date_calendar = QA_util_get_trade_range(start[:10],end[:10])
data = pd.DataFrame()
for date in date_calendar:
try:
intdate = QA_util_date_str2int(date)
file_temp = pd.read_csv(
os.path.join(path,
str(intdate) + '.csv'))
data = data.append(file_temp)
except:
pass
# data = data[data['Time']>=85900]
data['type'] = frequence
data['code'] = code
data = data.drop_duplicates(subset=['RealDate', 'Time'])
return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)(
DATASOURCE.COFUND, data)[start:end]
else:
print('当前仅支持1分钟的数据调用')
raise NotImplementedError
else:
print('当前仅支持L8的数据调用')
raise NotImplementedError
except Exception as exp:
print("code is ", code)
print(exp.__str__)
return None