def QA_etl_stock_calendar(type = "day", mark_day = str(datetime.date.today())):
if type == "all":
data = QA_fetch_stock_financial_calendar_adv(list(QA_fetch_stock_list_adv()['code'])).data.reset_index(drop=True)
QA_util_sql_store_mysql(data, "stock_calendar",if_exists='replace')
elif type == "day":
data = QA_fetch_stock_financial_calendar_adv(list(QA_fetch_stock_list_adv()['code']), mark_day).data
if data is None:
print("We have no calendar data for the day {}".format(str(datetime.date.today())))
else:
data = data.reset_index(drop=True)
QA_util_sql_store_mysql(data, "stock_calendar",if_exists='append')
def QA_etl_stock_divyield(type = "day", mark_day = str(datetime.date.today())):
if type == "all":
data = QA_fetch_stock_divyield_adv(list(QA_fetch_stock_list_adv()['code'])).data.reset_index()
QA_util_sql_store_mysql(data, "stock_divyield",if_exists='replace')
elif type == "day":
data = QA_fetch_stock_divyield_adv(list(QA_fetch_stock_list_adv()['code']), mark_day).data
if data is None:
print("We have no Divyield data for the day {}".format(str(datetime.date.today())))
else:
data = data.reset_index()
QA_util_sql_store_mysql(data, "stock_divyield",if_exists='append')
def QA_etl_stock_xdxr(type = "day", mark_day = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK XDXR ==== {}'.format(mark_day), ui_log)
if type == "all":
data = QA_fetch_stock_xdxr(list(QA_fetch_stock_list_adv()['code'])).reset_index(drop=True).fillna(0)
QA_util_sql_store_mysql(data, "stock_xdxr",if_exists='replace')
elif type == "day":
data = QA_fetch_stock_xdxr(list(QA_fetch_stock_list_adv()['code']), mark_day)
if data is None:
print("We have no XDXR data for the day {}".format(mark_day))
else:
data = data.reset_index(drop=True).fillna(0)
QA_util_sql_store_mysql(data, "stock_xdxr",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK XDXR HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_shares(ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK SHARES ==== {}'.format(str(datetime.date.today())), ui_log)
data = QA_fetch_stock_shares_adv(list(QA_fetch_stock_list_adv()['code'])).data
QA_util_sql_store_mysql(data, "stock_shares",if_exists='replace')
QA_util_log_info(
'##JOB ETL STOCK SHARES HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_divyield(type = "crawl", mark_day = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK divyield ==== {}'.format(mark_day), ui_log)
if type == "all":
data = QA_fetch_stock_divyield_adv(list(QA_fetch_stock_list_adv()['code']),start = "all").data.reset_index()
QA_util_sql_store_mysql(data, "stock_divyield",if_exists='replace')
elif type == "crawl":
data = QA_fetch_stock_divyield_adv(list(QA_fetch_stock_list_adv()['code']), mark_day).data
if data is None:
print("We have no Divyield data for the day {}".format(mark_day))
QA_util_log_info(
'##JOB NO STOCK Divyield HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
else:
data = data.reset_index()
QA_util_sql_store_mysql(data, "stock_divyield",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK Divyield HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_calendar(type = "crawl", start = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK CALENDAR ==== {}'.format(start), ui_log)
if type == "all":
data = QA_fetch_stock_financial_calendar_adv(list(QA_fetch_stock_list_adv()['code']),start = "all", type = 'report').data.reset_index(drop=True)
QA_util_sql_store_mysql(data, "stock_calendar",if_exists='replace')
elif type == "crawl":
data = QA_fetch_stock_financial_calendar_adv(list(QA_fetch_stock_list_adv()['code']), start, type = 'crawl').data
if data is None:
print("We have no calendar data for the day {}".format(start))
QA_util_log_info(
'##JOB NO STOCK CALENDAR HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
else:
data = data.reset_index(drop=True)
QA_util_sql_store_mysql(data, "stock_calendar",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK CALENDAR HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_financial_wy(type = "crawl", start_date = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK FINANCIAL REPORT WY ==== {}'.format(start_date), ui_log)
if type == 'all':
data = QA_fetch_financial_report_wy_adv(list(QA_fetch_stock_list_adv()['code'])).data.reset_index(drop=True).fillna(0)
QA_util_sql_store_mysql(data, "stock_financial_wy",if_exists='replace')
elif type == "crawl":
data = QA_fetch_financial_report_wy_adv(list(QA_fetch_stock_list_adv()['code']),start_date,type = 'crawl').data
if data is None:
print("We have no financial data for the day {}".format(str(datetime.date.today())))
QA_util_log_info(
'##JOB NO STOCK FINANCIAL REPORT WY HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
else:
data = data.reset_index(drop=True).fillna(0)
QA_util_sql_store_mysql(data, "stock_financial_wy",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK FINANCIAL REPORT WY HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_technical_day(type = "day", mark_day = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK TECHNICAL ==== {}'.format(mark_day), ui_log)
if type == "all":
data = QA_fetch_stock_technical_index_adv(list(QA_fetch_stock_list_adv()['code'])).data.reset_index()
QA_util_sql_store_mysql(data, "stock_technical",if_exists='replace')
elif type == "day":
data = QA_fetch_stock_technical_index_adv(list(QA_fetch_stock_list_adv()['code']), mark_day).data
if data is None:
print("We have no Technical data for the day {}".format(mark_day))
QA_util_log_info(
'##JOB NO STOCK TECHNICAL HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
else:
data = data.reset_index()
QA_util_sql_store_mysql(data, "stock_technical",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK TECHNICAL HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_SU_save_stock_shares_day(client=DATABASE,
ui_log=None,
ui_progress=None):
'''
save stock_day
保存财报日历
历史全部数据
:return:
'''
END_DATE = QA_util_today_str()
START_DATE = QA_util_datetime_to_strdate(
QA_util_add_days(QA_util_today_str(), -7))
code = list(QA_fetch_stock_list_adv()['code'])
stock_shares = client.stock_shares
stock_shares.create_index([("code", pymongo.ASCENDING),
("begin_date", pymongo.ASCENDING),
('total_shares', pymongo.DESCENDING),
('reason', pymongo.DESCENDING),
('send_date', pymongo.DESCENDING)],
unique=True)
err = []
def __saving_work(code, stock_shares):
try:
QA_util_log_info(
'##JOB01 Now Saving SSINA shares change==== {}'.format(
str(code)), ui_log)
stock_shares.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_shares_sina(code)),
ordered=False)
gc.collect()
except Exception as error0:
print(error0)
err.append(str(code))
for item in code:
QA_util_log_info('The {} of Total {}'.format((code.index(item) + 1),
len(code)))
strProgressToLog = 'DOWNLOAD PROGRESS {}'.format(
str(float((code.index(item) + 1) / len(code) * 100))[0:4] + '%',
ui_log)
intProgressToLog = int(float((code.index(item) + 1) / len(code) * 100))
QA_util_log_info(strProgressToLog,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intProgressToLog)
__saving_work(item, stock_shares)
if len(err) < 1:
QA_util_log_info('SUCCESS save SINA shares change ^_^', ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log)
QA_util_log_info(err, ui_log)
def QA_etl_stock_financial(type="day", mark_day=str(datetime.date.today())):
if type == 'all':
data = QA_fetch_financial_report_adv(
list(QA_fetch_stock_list_adv()['code'])).data.reset_index(
drop=True).drop("_id", 1).fillna(0)
QA_util_sql_store_mysql(data, "stock_financial", if_exists='replace')
elif type == "day":
data = QA_fetch_financial_report_adv(list(
QA_fetch_stock_list_adv()['code']),
mark_day,
type='date').data
print(data)
if data is None:
print("We have no financial data for the day {}".format(
str(datetime.date.today())))
else:
data = data.reset_index(drop=True).drop("_id", 1).fillna(0)
QA_util_sql_store_mysql(data,
"stock_financial",
if_exists='append')
def QA_SU_save_financial_report_day(client=DATABASE,
ui_log=None,
ui_progress=None):
'''
save stock_day
保存财报日历
历史全部数据
:return:
'''
code = list(
QA_fetch_stock_financial_calendar_adv(
list(QA_fetch_stock_list_adv()['code']),
QA_util_today_str()).data['code'])
stock_financial = client.stock_financial
stock_financial.create_index([("code", pymongo.ASCENDING),
("report_date", pymongo.ASCENDING)],
unique=True)
err = []
def __saving_work(code, stock_financial):
try:
QA_util_log_info(
'##JOB01 Now Saving THS financial_report==== {}'.format(
str(code)), ui_log)
stock_financial.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_report_ths(code)),
ordered=False)
gc.collect()
except Exception as error0:
print(error0)
err.append(str(code))
for item in code:
QA_util_log_info('The {} of Total {}'.format((code.index(item) + 1),
len(code)))
strProgressToLog = 'DOWNLOAD PROGRESS {}'.format(
str(float((code.index(item) + 1) / len(code) * 100))[0:4] + '%',
ui_log)
intProgressToLog = int(float((code.index(item) + 1) / len(code) * 100))
QA_util_log_info(strProgressToLog,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intProgressToLog)
__saving_work(item, stock_financial)
if len(err) < 1:
QA_util_log_info('SUCCESS save THS financial_report ^_^', ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log)
QA_util_log_info(err, ui_log)
def __init__(self, code=None, start=None, end=None, if_whole_market=True):
"""选股器
Keyword Arguments:
code {[type]} -- [description] (default: {None})
start {[type]} -- [description] (default: {None})
end {[type]} -- [description] (default: {None})
if_whole_market {bool} -- [description] (default: {True})
"""
self.code_list = QA_fetch_stock_list_adv().code.tolist(
) if code is None and if_whole_market else code
def QA_SU_save_financial_report_his(client=DATABASE,
ui_log=None,
ui_progress=None):
'''
save stock_day
保存财报日历
反向查询四个季度财报
:return:
'''
YEARS = list(
QA_util_getBetweenYear('2002-01-01', QA_util_today_str()).values())
code = list(QA_fetch_stock_list_adv()['code'])
stock_financial_sina = client.stock_financial_sina
stock_financial_sina.create_index([("code", pymongo.ASCENDING),
("report_date", pymongo.ASCENDING)],
unique=True)
err = []
def __saving_work(code, stock_financial_sina):
try:
QA_util_log_info(
'##JOB01 Now Saving SINA financial_report==== {}'.format(
str(code)), ui_log)
stock_financial_sina.insert_many(QA_util_to_json_from_pandas(
QA_fetch_get_stock_report_sina(code, YEARS)),
ordered=False)
except Exception as error0:
print(error0)
err.append(str(code))
for item in code:
QA_util_log_info('The {} of Total {}'.format((code.index(item) + 1),
len(code)))
strProgressToLog = 'DOWNLOAD PROGRESS {}'.format(
str(float((code.index(item) + 1) / len(code) * 100))[0:4] + '%',
ui_log)
intProgressToLog = int(float((code.index(item) + 1) / len(code) * 100))
QA_util_log_info(strProgressToLog,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intProgressToLog)
__saving_work(item, stock_financial_sina)
if len(err) < 1:
QA_util_log_info('SUCCESS save SINA financial_report ^_^', ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log)
QA_util_log_info(err, ui_log)
def QA_etl_stock_day(type = "day", mark_day = str(datetime.date.today()),ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK DAY ==== {}'.format(mark_day), ui_log)
codes = list(QA_fetch_stock_list_adv()['code'])
if type == "all":
data = ETL_stock_day(codes).reset_index()
QA_util_sql_store_mysql(data, "stock_market_day",if_exists='replace')
elif type == "day":
data = ETL_stock_day(codes, mark_day, mark_day).reset_index()
if data is None:
print("We have no MARKET data for the day {}".format(mark_day))
else:
QA_util_sql_store_mysql(data, "stock_market_day",if_exists='append')
QA_util_log_info(
'##JOB ETL STOCK DAY HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def get_quant_data(start_date, end_date, type = 'crawl', block = False, sub_block= True):
if block is True:
data = QA.QA_fetch_stock_block()
codes = list(data[data.blockname.isin(['上证50','沪深300','创业300','上证180','上证380','深证100','深证300','中证100','中证200'])]['code'].drop_duplicates())
#codes = [i for i in codes if i.startswith('300') == False]
else:
codes = list(QA_fetch_stock_list_adv()['code'])
if type == 'crawl':
res = QA_fetch_stock_quant_pre_adv(codes,start_date,end_date, block = sub_block).data
if type == 'model':
res = QA_fetch_get_quant_data(codes, start_date, end_date, block = sub_block).set_index(['date','code']).drop(['date_stamp'], axis=1)
target = QA_fetch_stock_target(codes, start_date, end_date)
res = res.join(target)
#res = res[(res['RNG_L_O'] <= 5 & res['LAG_TOR_O'] < 1)]
#dummy_industry = pd.get_dummies(res['INDUSTRY']).astype(float)
#dummy_industry.columns = ['I_' + i for i in list(dummy_industry.columns)]
#res = pd.concat([res[[col for col in list(res.columns) if col != 'INDUSTRY']],dummy_industry],axis = 1)
return(res)
def QA_SU_save_stock_quant_day(code=None,
start_date=None,
end_date=None,
ui_log=None,
ui_progress=None):
if start_date is None:
if end_date is None:
start_date = QA_util_get_pre_trade_date(QA_util_today_str(), 1)
end_date = QA_util_today_str()
elif end_date is not None:
start_date = '2008-01-01'
elif start_date is not None:
if end_date == None:
end_date = QA_util_today_str()
elif end_date is not None:
if end_date < start_date:
print('end_date should large than start_date')
if code is None:
code = list(QA_fetch_stock_list_adv()['code'])
financial = DATABASE.stock_quant_data_financial
financial.create_index([("code", ASCENDING), ("date_stamp", ASCENDING)],
unique=True)
index = DATABASE.stock_quant_data_index
index.create_index([("code", ASCENDING), ("date_stamp", ASCENDING)],
unique=True)
week = DATABASE.stock_quant_data_week
week.create_index([("code", ASCENDING), ("date_stamp", ASCENDING)],
unique=True)
alpha = DATABASE.stock_quant_data_alpha
alpha.create_index([("code", ASCENDING), ("date_stamp", ASCENDING)],
unique=True)
try:
data1 = QA_fetch_get_quant_data(code, start_date, end_date)
except:
data1 = None
else:
QA_util_log_info(
'##JOB got Data stock quant data ============== from {from_} to {to_} '
.format(from_=start_date, to_=end_date), ui_log)
deal_date_list = QA_util_get_trade_range(start_date, end_date)
if deal_date_list is None:
print('not a trading day')
elif data1 is None:
print('not a trading day')
else:
for deal_date in deal_date_list:
if QA_util_if_trade(deal_date):
data = data1[data1['date'] == deal_date]
else:
data = None
if data is not None:
data = data.drop_duplicates((['code', 'date']))
financial_data = data[[
'code', 'date', 'date_stamp', 'INDUSTRY', 'TOTAL_MARKET',
'TRA_RATE', 'DAYS', 'AVG5', 'AVG10', 'AVG20', 'AVG30',
'AVG60', 'LAG', 'LAG5', 'LAG10', 'LAG20', 'LAG30', 'LAG60',
'LAG_TOR', 'AVG5_TOR', 'AVG20_TOR', 'AVG30_TOR',
'AVG60_TOR', 'GROSSMARGIN', 'GROSSMARGIN_L2Y',
'GROSSMARGIN_L3Y', 'GROSSMARGIN_L4Y', 'GROSSMARGIN_LY',
'NETCASHOPERATINRATE', 'NETCASHOPERATINRATE_L2Y',
'NETCASHOPERATINRATE_L3Y', 'NETCASHOPERATINRATE_LY',
'NETPROFIT_INRATE', 'NETPROFIT_INRATE_L2Y',
'NETPROFIT_INRATE_L3Y', 'NETPROFIT_INRATE_LY',
'OPERATINGRINRATE', 'OPERATINGRINRATE_L2Y',
'OPERATINGRINRATE_L3Y', 'OPERATINGRINRATE_LY', 'PB', 'PBG',
'PC', 'PE', 'PEG', 'PM', 'PS', 'PSG', 'PT', 'RNG', 'RNG_L',
'RNG_5', 'RNG_10', 'RNG_20', 'RNG_30', 'RNG_60',
'AVG5_RNG', 'AVG10_RNG', 'AVG20_RNG', 'AVG30_RNG',
'AVG60_RNG', 'ROA', 'ROA_L2Y', 'ROA_L3Y', 'ROA_L4Y',
'ROA_LY', 'ROE', 'ROE_L2Y', 'ROE_L3Y', 'ROE_L4Y', 'ROE_LY',
'AVG5_CR', 'AVG10_CR', 'AVG20_CR', 'AVG30_CR', 'AVG60_CR',
'AVG5_TR', 'AVG10_TR', 'AVG20_TR', 'AVG30_TR', 'AVG60_TR',
'TOTALPROFITINRATE', 'TOTALPROFITINRATE_L2Y',
'TOTALPROFITINRATE_L3Y', 'TOTALPROFITINRATE_LY',
'PBRANK_10VAL', 'PBRANK_20VAL', 'PBRANK_30VAL',
'PBRANK_60VAL', 'PBRANK_90VAL', 'PB_10PCT', 'PB_10VAL',
'PB_20PCT', 'PB_20VAL', 'PB_30PCT', 'PB_30VAL', 'PB_60PCT',
'PB_60VAL', 'PB_90PCT', 'PB_90VAL', 'PEG_10PCT',
'PEG_10VAL', 'PEG_20PCT', 'PEG_20VAL', 'PEG_30PCT',
'PEG_30VAL', 'PEG_60PCT', 'PEG_60VAL', 'PEG_90PCT',
'PEG_90VAL', 'PERANK_10VAL', 'PERANK_20VAL',
'PERANK_30VAL', 'PERANK_60VAL', 'PERANK_90VAL', 'PE_10PCT',
'PE_10VAL', 'PE_20PCT', 'PE_20VAL', 'PE_30PCT', 'PE_30VAL',
'PE_60PCT', 'PE_60VAL', 'PE_90PCT', 'PE_90VAL', 'PS_10PCT',
'PS_10VAL', 'PS_20PCT', 'PS_20VAL', 'PS_30PCT', 'PS_30VAL',
'PS_60PCT', 'PS_60VAL', 'PS_90PCT', 'PS_90VAL', 'FINA_VAL',
'RNG60_RES', 'RNG20_RES', 'RNG_LO', 'LAG_TORO', 'DAYSO'
]]
alpha_data = data[[
'code', 'date', 'date_stamp', 'alpha_001', 'alpha_002',
'alpha_003', 'alpha_004', 'alpha_005', 'alpha_006',
'alpha_007', 'alpha_008', 'alpha_009', 'alpha_010',
'alpha_012', 'alpha_013', 'alpha_014', 'alpha_015',
'alpha_016', 'alpha_017', 'alpha_018', 'alpha_019',
'alpha_020', 'alpha_021', 'alpha_022', 'alpha_023',
'alpha_024', 'alpha_025', 'alpha_026', 'alpha_028',
'alpha_029', 'alpha_031', 'alpha_032', 'alpha_033',
'alpha_034', 'alpha_035', 'alpha_036', 'alpha_037',
'alpha_038', 'alpha_039', 'alpha_040', 'alpha_041',
'alpha_042', 'alpha_044', 'alpha_045', 'alpha_046',
'alpha_047', 'alpha_048', 'alpha_049', 'alpha_052',
'alpha_053', 'alpha_054', 'alpha_055', 'alpha_056',
'alpha_057', 'alpha_058', 'alpha_059', 'alpha_061',
'alpha_062', 'alpha_063', 'alpha_064', 'alpha_065',
'alpha_066', 'alpha_067', 'alpha_068', 'alpha_071',
'alpha_072', 'alpha_074', 'alpha_077', 'alpha_078',
'alpha_080', 'alpha_082', 'alpha_083', 'alpha_085',
'alpha_086', 'alpha_087', 'alpha_088', 'alpha_089',
'alpha_090', 'alpha_091', 'alpha_092', 'alpha_093',
'alpha_096', 'alpha_098', 'alpha_099', 'alpha_102',
'alpha_103', 'alpha_104', 'alpha_105', 'alpha_106',
'alpha_107', 'alpha_108', 'alpha_109', 'alpha_113',
'alpha_114', 'alpha_115', 'alpha_116', 'alpha_117',
'alpha_118', 'alpha_119', 'alpha_120', 'alpha_122',
'alpha_123', 'alpha_124', 'alpha_125', 'alpha_126',
'alpha_129', 'alpha_130', 'alpha_133', 'alpha_134',
'alpha_135', 'alpha_138', 'alpha_139', 'alpha_141',
'alpha_142', 'alpha_145', 'alpha_148', 'alpha_152',
'alpha_153', 'alpha_156', 'alpha_158', 'alpha_159',
'alpha_160', 'alpha_161', 'alpha_162', 'alpha_163',
'alpha_164', 'alpha_167', 'alpha_168', 'alpha_169',
'alpha_170', 'alpha_171', 'alpha_172', 'alpha_173',
'alpha_175', 'alpha_176', 'alpha_177', 'alpha_178',
'alpha_179', 'alpha_184', 'alpha_185', 'alpha_186',
'alpha_187', 'alpha_188', 'alpha_189', 'alpha_191'
]]
index_data = data[[
'code', 'date', 'date_stamp', 'AD', 'ADDI', 'ADDI_C',
'ADTM', 'ADX', 'ADXR', 'ADX_C', 'AD_C', 'AMA', 'ASI',
'ASIT', 'ATR', 'ATRR', 'BBI', 'BIAS1', 'BIAS2', 'BIAS3',
'BODY', 'BODY_ABS', 'BOLL', 'CCI', 'CHO', 'DDD', 'DDI',
'DDI_C', 'DEA', 'DI1', 'DI2', 'DIF', 'DI_M', 'KDJ_D',
'KDJ_J', 'KDJ_K', 'LB', 'MA1', 'MA10', 'MA120', 'MA180',
'MA2', 'MA20', 'MA3', 'MA4', 'MA5', 'MA60', 'MAADTM',
'MACD', 'MACHO', 'MAOSC', 'MAVPT', 'MFI', 'MFI_C',
'MIKE_BOLL', 'MR', 'MS', 'MTM', 'MTMMA', 'OBV', 'OBV_C',
'OSC', 'PRICE_PCG', 'ROC', 'ROCMA', 'RSI1', 'RSI1_C',
'RSI2', 'RSI2_C', 'RSI3', 'RSI3_C', 'RSV', 'SHA_LOW',
'SHA_UP', 'SKDJ_D', 'SKDJ_K', 'SR', 'SS', 'TR', 'UB',
'VPT', 'VR', 'VRSI', 'VRSI_C', 'VSTD', 'WIDTH', 'WR',
'WR1', 'WR2', 'WS', 'CCI_CROSS4', 'DMA_CROSS1',
'CDLMORNINGDOJISTAR', 'CDLSEPARATINGLINES', 'WR_CROSS1',
'KDJ_CROSS2', 'CDLHARAMICROSS', 'CDLEVENINGSTAR',
'BBI_CROSS2', 'VPT_CROSS1', 'CROSS_SC', 'CDLSHORTLINE',
'SKDJ_CROSS1', 'CDLABANDONEDBABY', 'CDL3STARSINSOUTH',
'CDLUNIQUE3RIVER', 'CDLKICKINGBYLENGTH', 'CDLHOMINGPIGEON',
'CDLTAKURI', 'CDL3BLACKCROWS', 'CDLSTICKSANDWICH',
'CDLTASUKIGAP', 'VPT_CROSS2', 'CDLSHOOTINGSTAR',
'CDLCONCEALBABYSWALL', 'WR_CROSS2', 'ADTM_CROSS1',
'BIAS_CROSS2', 'MTM_CROSS4', 'CCI_CROSS3', 'CDLHAMMER',
'CDLMARUBOZU', 'MACD_TR', 'CDL3INSIDE',
'CDLUPSIDEGAP2CROWS', 'MTM_CROSS1', 'CDLGRAVESTONEDOJI',
'KDJ_CROSS1', 'CDLMATHOLD', 'MIKE_TR', 'CDLLADDERBOTTOM',
'CDLMORNINGSTAR', 'OSC_CROSS2', 'OSC_CROSS4', 'ADX_CROSS2',
'DI_CROSS1', 'MTM_CROSS2', 'CDLDRAGONFLYDOJI',
'CCI_CROSS2', 'CDLSPINNINGTOP', 'CDLHIKKAKEMOD',
'DMA_CROSS2', 'MIKE_WRJC', 'CROSS_JC', 'OSC_CROSS3',
'RSI_CROSS1', 'MIKE_WSJC', 'MTM_CROSS3', 'CDLADVANCEBLOCK',
'BIAS_CROSS1', 'CDLCLOSINGMARUBOZU', 'CDL3OUTSIDE',
'VPT_CROSS3', 'CDLEVENINGDOJISTAR', 'CDL2CROWS',
'CDLHANGINGMAN', 'ADTM_CROSS2', 'CDLMATCHINGLOW',
'CDLHIKKAKE', 'CDLKICKING', 'CDLCOUNTERATTACK',
'CHO_CROSS1', 'CDLHARAMI', 'BBI_CROSS1', 'MIKE_WRSC',
'CDLINVERTEDHAMMER', 'CCI_CROSS1', 'CDLBREAKAWAY',
'CDLGAPSIDESIDEWHITE', 'DI_CROSS2', 'CDL3WHITESOLDIERS',
'CDLTRISTAR', 'CDLXSIDEGAP3METHODS', 'CDLPIERCING',
'VPT_CROSS4', 'CDLLONGLINE', 'CDLDOJI', 'CDLHIGHWAVE',
'CDLSTALLEDPATTERN', 'ADX_CROSS1', 'CDL3LINESTRIKE',
'CDLBELTHOLD', 'CDLINNECK', 'CDLONNECK', 'CDLRICKSHAWMAN',
'CDLTHRUSTING', 'CDLIDENTICAL3CROWS', 'SKDJ_CROSS2',
'CDLDOJISTAR', 'RSI_CROSS2', 'OSC_CROSS1',
'CDLRISEFALL3METHODS', 'CDLLONGLEGGEDDOJI', 'MIKE_WSSC',
'CDLDARKCLOUDCOVER', 'CHO_CROSS2', 'CDLENGULFING'
]]
week_data = data[[
'code', 'date', 'date_stamp', 'AD_WK', 'ADDI_WK',
'ADDI_C_WK', 'ADTM_WK', 'ADX_WK', 'ADXR_WK', 'ADX_C_WK',
'AD_C_WK', 'AMA_WK', 'ASI_WK', 'ASIT_WK', 'ATR_WK',
'ATRR_WK', 'BBI_WK', 'BIAS1_WK', 'BIAS2_WK', 'BIAS3_WK',
'BODY_WK', 'BODY_ABS_WK', 'BOLL_WK', 'CCI_WK', 'CHO_WK',
'DDD_WK', 'DDI_WK', 'DDI_C_WK', 'DEA_WK', 'DI1_WK',
'DI2_WK', 'DIF_WK', 'DI_M_WK', 'KDJ_D_WK', 'KDJ_J_WK',
'KDJ_K_WK', 'LB_WK', 'MA1_WK', 'MA10_WK', 'MA120_WK',
'MA180_WK', 'MA2_WK', 'MA20_WK', 'MA3_WK', 'MA4_WK',
'MA5_WK', 'MA60_WK', 'MAADTM_WK', 'MACD_WK', 'MACHO_WK',
'MAOSC_WK', 'MAVPT_WK', 'MFI_WK', 'MFI_C_WK',
'MIKE_BOLL_WK', 'MR_WK', 'MS_WK', 'MTM_WK', 'MTMMA_WK',
'OBV_WK', 'OBV_C_WK', 'OSC_WK', 'PRICE_PCG_WK', 'ROC_WK',
'ROCMA_WK', 'RSI1_WK', 'RSI1_C_WK', 'RSI2_WK', 'RSI2_C_WK',
'RSI3_WK', 'RSI3_C_WK', 'RSV_WK', 'SHA_LOW_WK',
'SHA_UP_WK', 'SKDJ_D_WK', 'SKDJ_K_WK', 'SR_WK', 'SS_WK',
'TR_WK', 'UB_WK', 'VPT_WK', 'VR_WK', 'VRSI_WK',
'VRSI_C_WK', 'VSTD_WK', 'WIDTH_WK', 'WR_WK', 'WR1_WK',
'WR2_WK', 'WS_WK', 'CDLDRAGONFLYDOJI_WK', 'MIKE_WRJC_WK',
'CDLRICKSHAWMAN_WK', 'MIKE_WSSC_WK', 'DI_CROSS2_WK',
'CDLHARAMI_WK', 'BBI_CROSS2_WK', 'VPT_CROSS2_WK',
'CDLBELTHOLD_WK', 'CDLHAMMER_WK', 'CDL3INSIDE_WK',
'CDLTRISTAR_WK', 'OSC_CROSS1_WK', 'CDLMARUBOZU_WK',
'CDLTASUKIGAP_WK', 'CDLSPINNINGTOP_WK',
'CDLDARKCLOUDCOVER_WK', 'CDL3BLACKCROWS_WK',
'BIAS_CROSS2_WK', 'OSC_CROSS3_WK', 'CHO_CROSS1_WK',
'CDLMORNINGSTAR_WK', 'ADX_CROSS2_WK', 'CDLINNECK_WK',
'ADTM_CROSS2_WK', 'MACD_TR_WK', 'CDLDOJI_WK',
'MTM_CROSS1_WK', 'CDLCOUNTERATTACK_WK', 'CDLLONGLINE_WK',
'KDJ_CROSS1_WK', 'CDLADVANCEBLOCK_WK', 'CDLHANGINGMAN_WK',
'KDJ_CROSS2_WK', 'ADX_CROSS1_WK', 'CDLMATHOLD_WK',
'CDLABANDONEDBABY_WK', 'WR_CROSS2_WK', 'MIKE_WRSC_WK',
'OSC_CROSS2_WK', 'CDLGAPSIDESIDEWHITE_WK', 'CROSS_JC_WK',
'MTM_CROSS4_WK', 'CDLSHOOTINGSTAR_WK', 'ADTM_CROSS1_WK',
'CDL3OUTSIDE_WK', 'CDLLONGLEGGEDDOJI_WK',
'CDL3LINESTRIKE_WK', 'CDLHIKKAKE_WK',
'CDLSTALLEDPATTERN_WK', 'MTM_CROSS2_WK', 'SKDJ_CROSS2_WK',
'CDLEVENINGDOJISTAR_WK', 'OSC_CROSS4_WK', 'CDLTAKURI_WK',
'CDLSHORTLINE_WK', 'CROSS_SC_WK', 'CDLMATCHINGLOW_WK',
'CCI_CROSS4_WK', 'MIKE_WSJC_WK', 'CDLHOMINGPIGEON_WK',
'VPT_CROSS1_WK', 'CDLCLOSINGMARUBOZU_WK', 'WR_CROSS1_WK',
'CDLTHRUSTING_WK', 'BBI_CROSS1_WK', 'DMA_CROSS2_WK',
'RSI_CROSS1_WK', 'CDLRISEFALL3METHODS_WK',
'CDLHIKKAKEMOD_WK', 'CCI_CROSS3_WK',
'CDLKICKINGBYLENGTH_WK', 'CDLLADDERBOTTOM_WK',
'DI_CROSS1_WK', 'VPT_CROSS3_WK', 'CDLHARAMICROSS_WK',
'CHO_CROSS2_WK', 'CCI_CROSS2_WK', 'CDL3STARSINSOUTH_WK',
'CDLXSIDEGAP3METHODS_WK', 'RSI_CROSS2_WK', 'MIKE_TR_WK',
'CDLDOJISTAR_WK', 'CDLCONCEALBABYSWALL_WK',
'CDLPIERCING_WK', 'CDLHIGHWAVE_WK',
'CDLMORNINGDOJISTAR_WK', 'CDLSTICKSANDWICH_WK',
'CDLGRAVESTONEDOJI_WK', 'CDLINVERTEDHAMMER_WK',
'CDLKICKING_WK', 'CDLSEPARATINGLINES_WK',
'CDLBREAKAWAY_WK', 'MTM_CROSS3_WK', 'CDLUNIQUE3RIVER_WK',
'CCI_CROSS1_WK', 'DMA_CROSS1_WK', 'VPT_CROSS4_WK',
'CDLEVENINGSTAR_WK', 'CDL2CROWS_WK',
'CDL3WHITESOLDIERS_WK', 'CDLIDENTICAL3CROWS_WK',
'CDLUPSIDEGAP2CROWS_WK', 'CDLENGULFING_WK',
'SKDJ_CROSS1_WK', 'BIAS_CROSS1_WK', 'CDLONNECK_WK'
]]
QA_util_log_info(
'##JOB01 Pre Data stock quant data ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
financial_res = QA_util_to_json_from_pandas(financial_data)
alpha_res = QA_util_to_json_from_pandas(alpha_data)
index_res = QA_util_to_json_from_pandas(index_data)
week_res = QA_util_to_json_from_pandas(week_data)
QA_util_log_info(
'##JOB02 Got Data stock quant data ============== {deal_date}'
.format(deal_date=deal_date), ui_log)
try:
financial.insert_many(financial_res, ordered=False)
QA_util_log_info(
'##JOB03 Now stock quant data financial saved ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
except Exception as e:
if isinstance(e, MemoryError):
financial.insert_many(financial_res, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
try:
alpha.insert_many(alpha_res, ordered=False)
QA_util_log_info(
'##JOB03 Now stock quant data alpha saved ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
except Exception as e:
if isinstance(e, MemoryError):
alpha.insert_many(alpha_res, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
try:
week.insert_many(week_res, ordered=False)
QA_util_log_info(
'##JOB03 Now stock quant data week saved ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
except Exception as e:
if isinstance(e, MemoryError):
week.insert_many(week_res, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
try:
index.insert_many(index_res, ordered=False)
QA_util_log_info(
'##JOB03 Now stock quant data index saved ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
except Exception as e:
if isinstance(e, MemoryError):
index.insert_many(index_res, ordered=True)
elif isinstance(e, pymongo.bulk.BulkWriteError):
pass
else:
QA_util_log_info(
'##JOB01 No Data stock_quant_datat ============== {deal_date} '
.format(deal_date=deal_date), ui_log)
def QA_etl_stock_list(ui_log= None):
QA_util_log_info(
'##JOB Now ETL STOCK LIST ==== {}'.format(str(datetime.date.today())), ui_log)
QA_util_sql_store_mysql(QA_fetch_stock_list_adv().reset_index(drop=True), "stock_list",if_exists='replace')
QA_util_log_info(
'##JOB ETL STOCK LIST HAS BEEN SAVED ==== {}'.format(str(datetime.date.today())), ui_log)
def QA_etl_stock_block():
data = QA_fetch_stock_block_adv(list(QA_fetch_stock_list_adv()['code'])).data.reset_index()
QA_util_sql_store_mysql(data, "stock_block",if_exists='replace')
def QA_etl_stock_financial(type = "all", mark_day = str(datetime.date.today())):
if type == 'all':
data = QA_fetch_financial_report_adv(list(QA_fetch_stock_list_adv()['code']),'1990-01-01',end=datetime.date.today()).data.reset_index(drop=True).drop("_id",1).fillna(0)
QA_util_sql_store_mysql(data, "stock_financial",if_exists='replace')
else:
raise DeprecationWarning('你的代码有点问题 我删掉了这部分')
def QA_etl_stock_list():
QA_util_sql_store_mysql(QA_fetch_stock_list_adv().reset_index(drop=True), "stock_list",if_exists='replace')
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(sleep)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
else:
print('Not Trading time {}'.format(_time))
time.sleep(sleep)
if __name__ == '__main__':
import time
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_list_adv
code = QA_fetch_stock_list_adv().code.tolist()
# DATABASE.realtime.create_index([('code', QA_util_sql_mongo_sort_ASCENDING),
# ('datetime', QA_util_sql_mongo_sort_ASCENDING)])
# print(len(code))
# x = QA_Tdx_Executor()
# print(x._queue.qsize())
# print(x.get_available())
# #data = x.get_security_bars(code[0], '15min', 20)
# # print(data)
# # for i in range(5):
# # print(x.get_realtime_concurrent(code))
# for i in range(100000):
# _time = datetime.datetime.now()
def QA_SU_save_stock_fianacial_percent_his(code=None,
start_date=None,
end_date=None,
client=DATABASE,
ui_log=None,
ui_progress=None):
if code is None:
codes = list(QA_fetch_stock_list_adv()['code'])
else:
codes = QA_util_code_tolist(code)
if start_date is None:
if end_date is None:
start_date = QA_util_today_str()
end_date = start_date
elif end_date is not None:
start_date = '2008-01-01'
elif start_date is not None:
if end_date == None:
end_date = QA_util_today_str()
elif end_date is not None:
if end_date < start_date:
print('end_date should large than start_date')
stock_financial_percent = DATABASE.stock_financial_percent
stock_financial_percent.create_index([("code", ASCENDING),
("date_stamp", ASCENDING)],
unique=True)
err = []
def __saving_work(code, START_DATE, END_DATE, stock_financial_percent):
try:
QA_util_log_info(
'##JOB01 Pre Data stock_fianacial_percent from {START_DATE} to {END_DATE} '
.format(START_DATE=START_DATE, END_DATE=END_DATE), ui_log)
data = QA_fetch_get_stock_financial_percent(
code, START_DATE, END_DATE)
data = data.drop_duplicates((['code', 'date']))
QA_util_log_info(
'##JOB02 Got Data stock_fianacial_percent from {START_DATE} to {END_DATE} '
.format(START_DATE=START_DATE, END_DATE=END_DATE), ui_log)
if data is not None:
stock_financial_percent.insert_many(
QA_util_to_json_from_pandas(data), ordered=False)
QA_util_log_info(
'##JOB03 Now stock_fianacial_percent saved from {START_DATE} to {END_DATE} '
.format(START_DATE=START_DATE, END_DATE=END_DATE), ui_log)
else:
QA_util_log_info(
'##JOB01 No Data stock_fianacial_percent from {START_DATE} to {END_DATE} '
.format(START_DATE=START_DATE, END_DATE=END_DATE), ui_log)
except Exception as error0:
print(error0)
err.append(str(code))
k = 500
for i in range(0, len(codes), k):
code = codes[i:i + k]
QA_util_log_info('The {} of Total {}'.format((i + k), len(codes)))
strProgressToLog = 'DOWNLOAD PROGRESS {}'.format(
str(float((i + k) / len(codes) * 100))[0:4] + '%', ui_log)
intProgressToLog = int(float((i + k) / len(codes) * 100))
QA_util_log_info(strProgressToLog,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intProgressToLog)
__saving_work(code, start_date, end_date, stock_financial_percent)
if len(err) < 1:
QA_util_log_info('SUCCESS save stock_fianacial_percent ^_^', ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log)
QA_util_log_info(err, ui_log)