def init_tdx_index_info(self, cdate=None, num=1):
def _set_index_info(cdate, code_id):
try:
if code_id in self.index_objs:
_obj = self.index_objs[code_id]
else:
_obj = CIndex(code_id) if code_id in list(
ct.TDX_INDEX_DICT.keys()) else TdxFgIndex(code_id)
return (code_id, _obj.set_k_data(cdate))
except Exception as e:
self.logger.error(e)
return (code_id, False)
#index_code_list = self.get_concerned_index_codes()
index_code_list = list(ct.TDX_INDEX_DICT.keys())
if cdate is None:
cfunc = partial(_set_index_info, cdate)
return concurrent_run(cfunc, index_code_list, num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_index_info, mdate)
if not concurrent_run(cfunc, index_code_list, num=5):
succeed = False
return succeed
def start_requests(self):
end_date = datetime.now().strftime('%Y-%m-%d')
start_date = self.get_nday_ago(end_date, 10, dformat = '%Y-%m-%d')
date_array = get_dates_array(start_date, end_date)
for mdate in date_array:
if CCalendar.is_trading_day(mdate, redis = self.redis):
up_url = self.get_url(self.LIMIT_UP, mdate)
yield Request(url=up_url, callback=self.parse, errback=self.errback_httpbin)
down_url = self.get_url(self.LIMIT_DOWN, mdate)
yield Request(url=down_url, callback=self.parse, errback=self.errback_httpbin)
def update(self, end_date=None, num=14):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis=self.redis):
if mdate == end_date or mdate in self.balcklist: continue
if not self.set_data(mdate):
succeed = False
return succeed
def update(self, end_date=None, num=3):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis=self.redis):
if not self.set_score(mdate):
succeed = False
self.logger.info("set score for %s set failed" % mdate)
return succeed
def update(self, end_date = datetime.now().strftime('%Y-%m-%d'), num = 30):
#if end_date == datetime.now().strftime('%Y-%m-%d'): end_date = get_day_nday_ago(end_date, num = 1, dformat = "%Y-%m-%d")
start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
succeed = True
for mdate in date_array:
if CCalendar.is_trading_day(mdate, redis = self.redis):
if not self.set_day_data(mdate):
self.logger.error("set %s data for rstock failed" % mdate)
succeed = False
return succeed
def update(self, end_date = None, num = 10):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
succeed = True
for mdate in date_array:
if CCalendar.is_trading_day(mdate, redis = self.redis):
if not self.crawl_data(mdate):
self.logger.error("%s set failed" % mdate)
succeed = False
return succeed
def update(self, end_date=datetime.now().strftime('%Y-%m-%d'), num=7):
succeed = True
base_df = self.stock_info_client.get_basics()
code_list = base_df.code.tolist()
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
if not self.set_r_financial_data(mdate, code_list):
self.logger.error("set %s data for rvaluation failed" %
mdate)
succeed = False
return succeed
def update(self, end_date=None, num=30):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
if end_date == datetime.now().strftime('%Y-%m-%d'):
end_date = get_day_nday_ago(end_date, num=1, dformat="%Y-%m-%d")
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
for mdate in get_dates_array(start_date, end_date):
if mdate in self.balcklist: continue
if CCalendar.is_trading_day(mdate, redis=self.redis):
if not self.set_k_data(mdate):
succeed = False
self.logger.info("market %s for %s set failed" %
(self.market, mdate))
return succeed
def update_index(self,
end_date=datetime.now().strftime('%Y-%m-%d'),
num=3361):
succeed = True
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date, asending=True)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
for code in ct.INDEX_DICT:
if not self.cval_client.set_index_valuation(code, mdate):
self.logger.error(
"{} set {} data for rvaluation failed".format(
code, mdate))
succeed = False
return succeed
def update_val(self,
end_date=datetime.now().strftime('%Y-%m-%d'),
num=7000):
succeed = True
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
for code in ct.INDEX_DICT:
if not self.set_index_valuation(code, mdate):
self.logger.error(
"%s set %s data for rvaluation failed" %
(code, mdate))
succeed = False
return succeed
def select_code(code_list, start_date, end_date):
date_arrays = list()
for mdate in get_dates_array(start_date, end_date, dformat = "%Y%m%d"):
if CCalendar.is_trading_day(transfer_int_to_date_string(mdate), redis = mredis):
date_arrays.append(mdate)
#choose stock which is not suspended verry long
total_df = pd.DataFrame()
for code in code_list:
df = ts.pro_bar(pro_api = ts_client, ts_code = add_suffix(code), adj = 'qfq', start_date = start_date, end_date = end_date)
if df is None: continue
if len(df) > int(0.8 * len(date_arrays)):
df = df.rename(columns = {"ts_code": "code", "trade_date": "date", "pct_change": "pchange"})
df = df.set_index('date')
total_df[code] = df.close
return total_df
def main():
dirname = '/Volumes/data/quant/stock/data/docs/blog/hellobiek.github.io/source/_posts'
unlock_path_ = "/Users/hellobiek/Documents/workspace/python/quant/smart_deal_tool/configure/futu.json"
futuTrader = FutuTrader(host=ct.FUTU_HOST_LOCAL,
port=ct.FUTU_PORT,
trd_env=TrdEnv.REAL,
market=ct.US_MARKET_SYMBOL,
unlock_path=unlock_path_)
start = '2019-05-28'
end = '2019-05-31'
orders = futuTrader.get_history_orders(start=start, end=end)
date_arrary = get_dates_array(start,
end,
dformat="%Y-%m-%d",
asending=True)
generate(orders, date_arrary, dirname, start, end)
def init_industry_info(self, cdate, num=1):
def _set_industry_info(cdate, code_id):
return (code_id, CIndex(code_id).set_k_data(cdate))
df = self.industry_info_client.get()
if cdate is None:
cfunc = partial(_set_industry_info, cdate)
return concurrent_run(cfunc, df.code.tolist(), num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_industry_info, mdate)
if not concurrent_run(cfunc, df.code.tolist(), num=5):
succeed = False
return succeed
def update(self, end_date=None, num=30):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
#start_date = "1997-12-30"
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
code_list = self.get_components(end_date)
if 0 == len(code_list):
self.logger.error("%s code_list for %s is empty" %
(end_date, self.index_code))
return False
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis=self.redis):
if not self.set_ratio(code_list, mdate):
self.logger.error("set %s score for %s set failed" %
(self.index_code, mdate))
succeed = False
return succeed
import const as ct
from cindex import CIndex
from jqdatasdk import auth
from ccalendar import CCalendar
from cstock_info import CStockInfo
from base.cdate import get_day_nday_ago, get_dates_array
if __name__ == '__main__':
num = 5500
end_date = '2019-08-13'
stock_info_client = CStockInfo()
df = stock_info_client.get()
code_list = df['code'].tolist()
name_list = df['name'].tolist()
code2namedict = dict(zip(code_list, name_list))
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
auth('18701683341', '52448oo78')
for code in [
'000001', '000016', '000300', '000905', '399001', '399005',
'399673'
]:
obj = CIndex(code)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
table_name = obj.get_components_table_name(mdate)
if obj.is_table_exists(table_name):
obj.mysql_client.delete(table_name)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
if not obj.set_components_data_from_joinquant(
