def get_k_data_in_range(self, start_date, end_date): ndays = delta_days(start_date, end_date) date_dmy_format = time.strftime("%m/%d/%Y", time.strptime(start_date, "%Y-%m-%d")) data_times = pd.date_range(date_dmy_format, periods=ndays, freq='D') date_only_array = np.vectorize(lambda s: s.strftime('%Y-%m-%d'))( data_times.to_pydatetime()) data_dict = OrderedDict() for _date in date_only_array: if CCalendar.is_trading_day(_date, redis=self.redis): table_name = self.get_table_name(_date) if table_name not in data_dict: data_dict[table_name] = list() data_dict[table_name].append(str(_date)) all_df = pd.DataFrame() for key in data_dict: table_list = sorted(data_dict[key], reverse=False) if len(table_list) == 1: df = self.get_data(table_list[0]) if df is not None: all_df = all_df.append(df) else: start_date = table_list[0] end_date = table_list[len(table_list) - 1] df = self.get_data_between(start_date, end_date) if df is not None: all_df = all_df.append(df) return all_df
def start_requests(self): end_date = datetime.now().strftime('%Y-%m-%d') start_date = self.get_nday_ago(end_date, 10, dformat = '%Y-%m-%d') date_array = get_dates_array(start_date, end_date) for mdate in date_array: if CCalendar.is_trading_day(mdate, redis = self.redis): up_url = self.get_url(self.LIMIT_UP, mdate) yield Request(url=up_url, callback=self.parse, errback=self.errback_httpbin) down_url = self.get_url(self.LIMIT_DOWN, mdate) yield Request(url=down_url, callback=self.parse, errback=self.errback_httpbin)
def update(self, end_date = None, num = 3): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") succeed = True for mdate in get_dates_array(start_date, end_date): if CCalendar.is_trading_day(mdate, redis = self.redis): if not self.set_score(mdate): succeed = False self.logger.info("set score for %s set failed" % mdate) return succeed
def update(self, end_date=None, num=10): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") succeed = True for mdate in get_dates_array(start_date, end_date): if CCalendar.is_trading_day(mdate, redis=self.redis): if mdate == end_date or mdate in self.balcklist: continue if not self.set_data(mdate): succeed = False return succeed
def update(self, end_date = None, num = 10): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") date_array = get_dates_array(start_date, end_date) succeed = True for mdate in date_array: if CCalendar.is_trading_day(mdate, redis = self.redis): if not self.crawl_data(mdate): self.logger.error("%s set failed" % mdate) succeed = False return succeed
def update(self, end_date = datetime.now().strftime('%Y-%m-%d'), num = 30): #if end_date == datetime.now().strftime('%Y-%m-%d'): end_date = get_day_nday_ago(end_date, num = 1, dformat = "%Y-%m-%d") start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") date_array = get_dates_array(start_date, end_date) succeed = True for mdate in date_array: if CCalendar.is_trading_day(mdate, redis = self.redis): if not self.set_day_data(mdate): self.logger.error("set %s data for rstock failed" % mdate) succeed = False return succeed
def update(self, end_date = None, num = 10): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') if end_date == datetime.now().strftime('%Y-%m-%d'): end_date = get_day_nday_ago(end_date, num = 1, dformat = "%Y-%m-%d") start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") succeed = True for mdate in get_dates_array(start_date, end_date): if mdate in self.balcklist: continue if CCalendar.is_trading_day(mdate, redis = self.redis): if not self.set_k_data(mdate): succeed = False self.logger.info("market %s for %s set failed" % (self.market, mdate)) return succeed
def update(self, end_date=datetime.now().strftime('%Y-%m-%d'), num=7): succeed = True base_df = self.stock_info_client.get_basics() code_list = base_df.code.tolist() start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") date_array = get_dates_array(start_date, end_date) for mdate in date_array: if CCalendar.is_trading_day(mdate): if not self.set_r_financial_data(mdate, code_list): self.logger.error("set %s data for rvaluation failed" % mdate) succeed = False return succeed
class DataPreparer: def __init__(self): self.logger = getLogger(__name__) self.cal_client = CCalendar( dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1', filepath='/Volumes/data/quant/stock/conf/calAll.csv') def is_collecting_time(self): now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] aft_open_hour, aft_open_minute, aft_open_second = (16, 00, 00) aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute, aft_open_second) aft_close_hour, aft_close_minute, aft_close_second = (22, 00, 00) aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute, aft_close_second) #self.logger.info("collecting now time. open_time:%s < now_time:%s < close_time:%s" % (aft_open_time, now_time, aft_close_time)) return aft_open_time < now_time < aft_close_time def prepare_data(self, cmds, timeout=2700): kill = lambda process: process.kill() cmd_list = list() for cmd in cmds: cmd_list.append(subprocess.Popen(cmd, shell=True)) my_timer = Timer(timeout, kill, cmd_list) try: my_timer.start() for cmd in cmd_list: cmd.communicate() finally: my_timer.cancel() def update(self, sleep_time): while True: try: self.logger.debug("enter update") if self.cal_client.is_trading_day(): if self.is_collecting_time(): ndate = get_latest_data_date( filepath= "/Volumes/data/quant/stock/data/stockdatainfo.json" ) mdate = transfer_date_string_to_int( datetime.now().strftime('%Y-%m-%d')) if ndate < mdate: self.prepare_data([SCRIPT1, SCRIPT2]) except Exception as e: self.logger.error(e) time.sleep(sleep_time)
def update(self, end_date = None, num = 30): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') #start_date = "1997-12-30" start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") succeed = True code_list = self.get_components(end_date) if 0 == len(code_list): self.logger.error("%s code_list for %s is empty" % (end_date, self.index_code)) return False for mdate in get_dates_array(start_date, end_date): if CCalendar.is_trading_day(mdate, redis = self.redis): if not self.set_ratio(code_list, mdate): self.logger.error("set %s score for %s set failed" % (self.index_code, mdate)) succeed = False return succeed
def update_index(self, end_date=datetime.now().strftime('%Y-%m-%d'), num=3361): succeed = True start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") date_array = get_dates_array(start_date, end_date, asending=True) for mdate in date_array: if CCalendar.is_trading_day(mdate): for code in ct.INDEX_DICT: if not self.cval_client.set_index_valuation(code, mdate): self.logger.error( "{} set {} data for rvaluation failed".format( code, mdate)) succeed = False return succeed
def update_val(self, end_date=datetime.now().strftime('%Y-%m-%d'), num=7000): succeed = True start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") date_array = get_dates_array(start_date, end_date) for mdate in date_array: if CCalendar.is_trading_day(mdate): for code in ct.INDEX_DICT: if not self.set_index_valuation(code, mdate): self.logger.error( "%s set %s data for rvaluation failed" % (code, mdate)) succeed = False return succeed
def select_code(code_list, start_date, end_date): date_arrays = list() for mdate in get_dates_array(start_date, end_date, dformat = "%Y%m%d"): if CCalendar.is_trading_day(transfer_int_to_date_string(mdate), redis = mredis): date_arrays.append(mdate) #choose stock which is not suspended verry long total_df = pd.DataFrame() for code in code_list: df = ts.pro_bar(pro_api = ts_client, ts_code = add_suffix(code), adj = 'qfq', start_date = start_date, end_date = end_date) if df is None: continue if len(df) > int(0.8 * len(date_arrays)): df = df.rename(columns = {"ts_code": "code", "trade_date": "date", "pct_change": "pchange"}) df = df.set_index('date') total_df[code] = df.close return total_df
def set_data(self, cdate=datetime.now().strftime('%Y-%m-%d')): if not CCalendar.is_trading_day(cdate, redis=self.redis): return False table_name = self.get_table_name(cdate) if not self.is_table_exists(table_name): if not self.create_table(table_name): self.logger.error("create rindex table failed") return False self.redis.sadd(self.dbname, table_name) if self.is_date_exists(table_name, cdate): self.logger.debug("existed rindex table:%s, date:%s" % (table_name, cdate)) return True df = self.generate_data(cdate) if df.empty: return False self.redis.set(ct.TODAY_ALL_INDUSTRY, _pickle.dumps(df, 2)) if self.mysql_client.set(df, table_name): return self.redis.sadd(table_name, cdate) return False
class DataPreparer: def __init__(self): self.logger = getLogger(__name__) self.cal_client = CCalendar(dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1', filepath = '/Volumes/data/quant/stock/conf/calAll.csv') def is_collecting_time(self): now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y,m,d = time.strptime(_date, "%Y-%m-%d")[0:3] aft_open_hour,aft_open_minute,aft_open_second = (16,00,00) aft_open_time = datetime(y,m,d,aft_open_hour,aft_open_minute,aft_open_second) aft_close_hour,aft_close_minute,aft_close_second = (22,00,00) aft_close_time = datetime(y,m,d,aft_close_hour,aft_close_minute,aft_close_second) #self.logger.info("collecting now time. open_time:%s < now_time:%s < close_time:%s" % (aft_open_time, now_time, aft_close_time)) return aft_open_time < now_time < aft_close_time def run(self, cmd, timeout): proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) timer = Timer(timeout, proc.kill) try: timer.start() stdout, stderr = proc.communicate() self.logger.debug("run cmd %s stdout:%s, stderr:%s" % (cmd, stdout.decode(), stderr.decode())) finally: timer.cancel() def update(self, sleep_time): while True: try: self.logger.debug("enter update") if self.cal_client.is_trading_day(): if self.is_collecting_time(): ndate = get_latest_data_date(filepath = "/Volumes/data/quant/stock/data/stockdatainfo.json") mdate = transfer_date_string_to_int(datetime.now().strftime('%Y-%m-%d')) if ndate < mdate: self.run(SCRIPT1, timeout = 600) self.run(SCRIPT2, timeout = 2700) except Exception as e: self.logger.error(e) time.sleep(sleep_time)
class DataManager: def __init__(self, dbinfo): self.combination_objs = dict() self.stock_objs = dict() self.evt = AsyncResult() self.dbinfo = dbinfo self.cal_client = CCalendar(dbinfo) self.comb_info_client = CombinationInfo(dbinfo) self.stock_info_client = CStockInfo(dbinfo) self.delisted_info_client = CDelisted(dbinfo) self.animation_client = CAnimation(dbinfo) self.subscriber = Subscriber() def is_collecting_time(self, now_time=None): if now_time is None: now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] mor_open_hour, mor_open_minute, mor_open_second = (19, 0, 0) mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute, mor_open_second) mor_close_hour, mor_close_minute, mor_close_second = (23, 59, 59) mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute, mor_close_second) return mor_open_time < now_time < mor_close_time def is_tcket_time(self, now_time=None): if now_time is None: now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] mor_open_hour, mor_open_minute, mor_open_second = (0, 0, 0) mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute, mor_open_second) mor_close_hour, mor_close_minute, mor_close_second = (9, 0, 0) mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute, mor_close_second) aft_open_hour, aft_open_minute, aft_open_second = (15, 10, 0) aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute, aft_open_second) aft_close_hour, aft_close_minute, aft_close_second = (23, 59, 59) aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute, aft_close_second) return (mor_open_time < now_time < mor_close_time) or (aft_open_time < now_time < aft_close_time) def collect(self, sleep_time): while True: try: self.init_all_stock_tick() except Exception as e: logger.error(e) time.sleep(sleep_time) def collect_combination_runtime_data(self): obj_pool = Pool(10) for code_id in self.combination_objs: try: if obj_pool.full(): obj_pool.join() obj_pool.spawn(self.combination_objs[code_id].run) except Exception as e: logger.info(e) obj_pool.join() obj_pool.kill() def collect_stock_runtime_data(self): obj_pool = Pool(100) for code_id in self.stock_objs: try: if obj_pool.full(): obj_pool.join() ret, df = self.subscriber.get_tick_data(add_prifix(code_id)) if 0 == ret: df = df.set_index('time') df.index = pd.to_datetime(df.index) obj_pool.spawn(self.stock_objs[code_id].run, df) except Exception as e: logger.info(e) obj_pool.join() obj_pool.kill() def run(self, sleep_time): while True: try: if self.cal_client.is_trading_day(): if is_trading_time() and not self.subscriber.status(): self.subscriber.start() self.init_combination_info() self.init_real_stock_info() elif is_trading_time() and self.subscriber.status(): self.collect_stock_runtime_data() self.collect_combination_runtime_data() self.animation_client.collect() elif not is_trading_time() and self.subscriber.status(): self.subscriber.stop() except Exception as e: logger.error(e) #traceback.print_exc() time.sleep(sleep_time) def update(self, sleep_time): while True: try: if self.cal_client.is_trading_day(): if self.is_collecting_time(): self.init() time.sleep(sleep_time) except Exception as e: logger.error(e) #traceback.print_exc() def init(self, status=False): self.cal_client.init(status) self.comb_info_client.init() self.stock_info_client.init() self.delisted_info_client.init(status) self.init_today_stock_tick() #self.halted_info_client.init(status) def get_concerned_list(self): combination_info = self.comb_info_client.get() if combination_info is None: return list() combination_info = combination_info.reset_index(drop=True) res_list = list() for index, _ in combination_info['code'].iteritems(): objliststr = combination_info.loc[index]['content'] objlist = objliststr.split(',') res_list.extend(objlist) return list(set(res_list)) def init_combination_info(self): trading_info = self.comb_info_client.get() for _, code_id in trading_info['code'].iteritems(): if str(code_id) not in self.combination_objs: self.combination_objs[str(code_id)] = Combination( self.dbinfo, code_id) def init_today_stock_tick(self): _date = datetime.now().strftime('%Y-%m-%d') obj_pool = Pool(50) df = self.stock_info_client.get() if self.cal_client.is_trading_day(_date): for _, code_id in df.code.iteritems(): _obj = self.stock_objs[ code_id] if code_id in self.stock_objs else CStock( self.dbinfo, code_id) try: if obj_pool.full(): obj_pool.join() obj_pool.spawn(_obj.set_ticket, _date) obj_pool.spawn(_obj.set_k_data) except Exception as e: logger.info(e) obj_pool.join() obj_pool.kill() def init_all_stock_tick(self): start_date = '2015-01-01' _today = datetime.now().strftime('%Y-%m-%d') num_days = delta_days(start_date, _today) start_date_dmy_format = time.strftime( "%m/%d/%Y", time.strptime(start_date, "%Y-%m-%d")) data_times = pd.date_range(start_date_dmy_format, periods=num_days, freq='D') date_only_array = np.vectorize(lambda s: s.strftime('%Y-%m-%d'))( data_times.to_pydatetime()) date_only_array = date_only_array[::-1] obj_pool = Pool(4) df = self.stock_info_client.get() for _, code_id in df.code.iteritems(): _obj = self.stock_objs[ code_id] if code_id in self.stock_objs else CStock( self.dbinfo, code_id) for _date in date_only_array: if self.cal_client.is_trading_day(_date): try: if obj_pool.full(): obj_pool.join() obj_pool.spawn(_obj.set_ticket, _date) except Exception as e: logger.info(e) obj_pool.join() obj_pool.kill() def init_real_stock_info(self): concerned_list = self.get_concerned_list() for code_id in concerned_list: ret = self.subscriber.subscribe_tick(add_prifix(code_id), CStock) if 0 == ret: if code_id not in self.stock_objs: self.stock_objs[code_id] = CStock(self.dbinfo, code_id) def download_and_extract(self, sleep_time): while True: try: if self.cal_client.is_trading_day(): if self.is_collecting_time(): download(ct.ZIP_DIR) list_files = os.listdir(ct.ZIP_DIR) for filename in list_files: if not filename.startswith('.'): file_path = os.path.join(ct.ZIP_DIR, filename) if os.path.exists(file_path): unzip(file_path, ct.TIC_DIR) except Exception as e: logger.error(e) time.sleep(sleep_time)
class DataManager: def __init__(self, dbinfo=ct.DB_INFO, redis_host=None): self.dbinfo = dbinfo self.logger = getLogger(__name__) self.index_objs = dict() self.stock_objs = dict() self.updating_date = None self.combination_objs = dict() self.cal_client = CCalendar(dbinfo, redis_host) self.index_info_client = IndexInfo() self.cvaluation_client = CValuation() self.reviewer = CReivew(dbinfo, redis_host) self.comb_info_client = CombinationInfo(dbinfo, redis_host) self.stock_info_client = CStockInfo(dbinfo, redis_host) self.rindex_stock_data_client = RIndexStock(dbinfo, redis_host) self.industry_info_client = IndustryInfo(dbinfo, redis_host) self.rindustry_info_client = RIndexIndustryInfo(dbinfo, redis_host) self.animation_client = CAnimation(dbinfo, redis_host) self.subscriber = Subscriber() self.quote_handler = StockQuoteHandler() self.ticker_handler = TickerHandler() self.connect_client = StockConnect(market_from=ct.SH_MARKET_SYMBOL, market_to=ct.HK_MARKET_SYMBOL, dbinfo=dbinfo, redis_host=redis_host) self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host) self.emotion_client = Emotion(dbinfo=dbinfo, redis_host=redis_host) self.sh_exchange_client = StockExchange(ct.SH_MARKET_SYMBOL) self.sz_exchange_client = StockExchange(ct.SZ_MARKET_SYMBOL) def is_collecting_time(self): now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] aft_open_hour, aft_open_minute, aft_open_second = (17, 10, 00) aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute, aft_open_second) aft_close_hour, aft_close_minute, aft_close_second = (23, 59, 59) aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute, aft_close_second) #self.logger.info("collecting now time. open_time:%s < now_time:%s < close_time:%s" % (aft_open_time, now_time, aft_close_time)) return aft_open_time < now_time < aft_close_time def is_morning_time(self, now_time=datetime.now()): _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] mor_open_hour, mor_open_minute, mor_open_second = (0, 0, 0) mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute, mor_open_second) mor_close_hour, mor_close_minute, mor_close_second = (6, 30, 0) mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute, mor_close_second) return mor_open_time < now_time < mor_close_time def collect_combination_runtime_data(self): def _combination_run(code_id): self.combination_objs[code_id].run() return (code_id, True) todo_iplist = list(self.combination_objs.keys()) return concurrent_run(_combination_run, todo_iplist, num=10) def collect_stock_runtime_data(self): if self.ticker_handler.empty(): return datas = self.ticker_handler.getQueue() while not datas.empty(): df = datas.get() df = df.set_index('time') df.index = pd.to_datetime(df.index) for code_str in set(df.code): code_id = code_str.split('.')[1] self.stock_objs[code_id].run(df.loc[df.code == code_str]) def init_real_stock_info(self): concerned_list = self.comb_info_client.get_concerned_list() prefix_concerned_list = [add_prifix(code) for code in concerned_list] ret = self.subscriber.subscribe(prefix_concerned_list, SubType.TICKER, self.ticker_handler) if 0 == ret: for code in concerned_list: if code not in self.stock_objs: self.stock_objs[code] = CStock(code, self.dbinfo, should_create_influxdb=True, should_create_mysqldb=True) return ret def init_index_info(self): index_list = ct.INDEX_DICT.keys() prefix_index_list = [add_index_prefix(code) for code in index_list] ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE, self.quote_handler) if 0 != ret: self.logger.error("subscribe for index list failed") return ret for code in index_list: if code not in self.index_objs: self.index_objs[code] = CIndex(code, should_create_influxdb=True, should_create_mysqldb=True) return 0 def collect_index_runtime_data(self): if self.quote_handler.empty(): return datas = self.quote_handler.getQueue() while not datas.empty(): df = datas.get() df['time'] = df.data_date + ' ' + df.data_time df = df.drop(['data_date', 'data_time'], axis=1) df = df.set_index('time') df.index = pd.to_datetime(df.index) for code_str in set(df.code): code_id = code_str.split('.')[1] self.index_objs[code_id].run(df.loc[df.code == code_str]) def run(self, sleep_time): while True: try: self.logger.debug("enter run") if self.cal_client.is_trading_day(): if is_trading_time(): t_sleep_time = 1 if not self.subscriber.status(): self.subscriber.start() if 0 == self.init_index_info( ) and 0 == self.init_real_stock_info(): self.init_combination_info() else: self.logger.debug("enter stop subscriber") self.subscriber.stop() else: self.collect_stock_runtime_data() self.collect_combination_runtime_data() self.collect_index_runtime_data() self.animation_client.collect() else: t_sleep_time = sleep_time if self.subscriber.status(): self.subscriber.stop() else: t_sleep_time = sleep_time except Exception as e: #traceback.print_exc() self.logger.error(e) gevent.sleep(t_sleep_time) def set_update_info(self, step_length, exec_date, cdate=None, filename=ct.STEPFILE): step_info = dict() if cdate is None: cdate = 'none' step_info[cdate] = dict() step_info[cdate]['step'] = step_length step_info[cdate]['date'] = exec_date with open(filename, 'w') as f: json.dump(step_info, f) self.logger.info("finish step :%s" % step_length) def get_update_info(self, cdate=None, exec_date=None, filename=ct.STEPFILE): if cdate is None: cdate = 'none' if not os.path.exists(filename): return (0, exec_date) with open(filename, 'r') as f: step_info = json.load(f) if cdate not in step_info: return (0, exec_date) return (step_info[cdate]['step'], step_info[cdate]['date']) def bootstrap(self, cdate=None, exec_date=datetime.now().strftime('%Y-%m-%d'), ndays=3): finished_step, exec_date = self.get_update_info(cdate, exec_date) self.logger.info("enter updating.%s" % finished_step) if finished_step < 1: if not self.cal_client.init(): self.logger.error("cal client init failed") return False self.set_update_info(1, exec_date, cdate) if finished_step < 2: if not self.index_info_client.update(): self.logger.error("index info init failed") return False self.set_update_info(2, exec_date, cdate) if finished_step < 3: if not self.stock_info_client.update(): self.logger.error("stock info init failed") return False self.set_update_info(3, exec_date, cdate) if finished_step < 4: if not self.comb_info_client.update(): self.logger.error("comb info init failed") return False self.set_update_info(4, exec_date, cdate) if finished_step < 5: if not self.industry_info_client.update(): self.logger.error("industry info init failed") return False self.set_update_info(5, exec_date, cdate) if finished_step < 6: if not self.init_tdx_index_info(cdate): self.logger.error("init tdx index info failed") return False self.set_update_info(6, exec_date, cdate) if finished_step < 7: if not self.sh_exchange_client.update(exec_date, num=ndays): self.logger.error("sh exchange update failed") return False self.set_update_info(7, exec_date, cdate) if finished_step < 8: if not self.sz_exchange_client.update(exec_date, num=ndays): self.logger.error("sz exchange update failed") return False self.set_update_info(8, exec_date, cdate) if finished_step < 9: if not self.init_index_components_info(exec_date): self.logger.error("init index components info failed") return False self.set_update_info(9, exec_date, cdate) if finished_step < 10: if not self.init_industry_info(cdate): self.logger.error("init industry info failed") return False self.set_update_info(10, exec_date, cdate) if finished_step < 11: if not self.rindustry_info_client.update(exec_date, num=ndays): self.logger.error("init %s rindustry info failed" % exec_date) return False self.set_update_info(11, exec_date, cdate) if finished_step < 12: if not self.init_yesterday_hk_info(exec_date, num=ndays): self.logger.error("init yesterday hk info failed") return False self.set_update_info(12, exec_date, cdate) if finished_step < 13: if not self.margin_client.update(exec_date, num=ndays): self.logger.error("init yesterday margin failed") return False self.set_update_info(13, exec_date, cdate) if finished_step < 14: if not self.init_stock_info(cdate): self.logger.error("init stock info set failed") return False self.set_update_info(14, exec_date, cdate) if finished_step < 15: if not self.init_base_float_profit(): self.logger.error("init base float profit for all stock") return False self.set_update_info(15, exec_date, cdate) if finished_step < 16: if not self.init_valuation_info(cdate): self.logger.error("init stock valuation info failed") return False self.set_update_info(16, exec_date, cdate) if finished_step < 17: if not self.init_rvaluation_info(cdate): self.logger.error("init r stock valuation info failed") return False self.set_update_info(17, exec_date, cdate) if finished_step < 18: if not self.init_rindex_valuation_info(cdate): self.logger.error("init r index valuation info failed") return False self.set_update_info(18, exec_date, cdate) if finished_step < 19: if not self.rindex_stock_data_client.update(exec_date, num=ndays): self.logger.error("rstock data set failed") return False self.set_update_info(19, exec_date, cdate) if finished_step < 20: if not self.set_bull_stock_ratio(exec_date, num=ndays): self.logger.error("bull ratio set failed") return False self.set_update_info(20, exec_date, cdate) self.logger.info("updating succeed") return True def clear_network_env(self): kill_process("google-chrome") kill_process("renderer") kill_process("Xvfb") kill_process("zygote") kill_process("defunct") kill_process("show-component-extension-options") def update(self, sleep_time): succeed = False while True: self.logger.debug("enter daily update process. %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) try: if self.cal_client.is_trading_day(): #self.logger.info("is trading day. %s, succeed:%s" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), succeed)) if self.is_collecting_time(): self.logger.debug( "enter collecting time. %s, succeed:%s" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), succeed)) if not succeed: self.clear_network_env() mdate = datetime.now().strftime('%Y-%m-%d') ndate = get_latest_data_date() if ndate is not None: if ndate >= transfer_date_string_to_int(mdate): if self.updating_date is None: self.updating_date = mdate succeed = self.bootstrap( cdate=self.updating_date, exec_date=self.updating_date) if succeed: self.updating_date = None else: self.logger.debug("%s is older for %s" % (ndate, mdate)) else: succeed = False gevent.sleep(sleep_time) except Exception as e: time.sleep(1) self.logger.error(e) def init_combination_info(self): trading_info = self.comb_info_client.get() for _, code_id in trading_info['code'].iteritems(): if str(code_id) not in self.combination_objs: self.combination_objs[str(code_id)] = Combination( code_id, self.dbinfo) def init_base_float_profit(self): def _set_base_float_profit(code_id): if CStock(code_id).set_base_floating_profit(): self.logger.info("%s set base float profit success" % code_id) return (code_id, True) else: self.logger.error("%s set base float profit failed" % code_id) return (code_id, False) df = self.stock_info_client.get() if df.empty: return False failed_list = df.code.tolist() return process_concurrent_run(_set_base_float_profit, failed_list, num=8) def init_rindex_valuation_info(self, cdate): for code in ct.INDEX_DICT: if not self.cvaluation_client.set_index_valuation(code, cdate): self.logger.error( "{} set {} data for rvaluation failed".format(code, mdate)) return False return True def init_rvaluation_info(self, cdate=None): def cget(mdate, code): return code, CStock(code).get_val_data(mdate) df = self.stock_info_client.get() code_list = df.code.tolist() try: obj_pool = Pool(5000) all_df = pd.DataFrame() cfunc = partial(cget, cdate) for code_data in obj_pool.imap_unordered(cfunc, code_list): if code_data[1] is not None and not code_data[1].empty: tem_df = code_data[1] tem_df['code'] = code_data[0] all_df = all_df.append(tem_df) obj_pool.join(timeout=5) obj_pool.kill() all_df = all_df.reset_index(drop=True) file_name = "{}.csv".format(cdate) file_path = Path(ct.RVALUATION_DIR) / file_name all_df.to_csv(file_path, index=False, header=True, mode='w', encoding='utf8') return True except Exception as e: self.logger.error(e) return False def init_valuation_info(self, cdate=None): df = self.stock_info_client.get() code_list = df['code'].tolist() time2market_list = df['timeToMarket'].tolist() code2timedict = dict(zip(code_list, time2market_list)) cfun = partial(self.cvaluation_client.set_stock_valuation, code2timedict, cdate) return process_concurrent_run(cfun, code_list, num=15, black_list=list()) def init_stock_info(self, cdate=None): def _set_stock_info(mdate, bonus_info, index_info, code_id): try: if CStock(code_id).set_k_data(bonus_info, index_info, mdate): self.logger.info("%s set k data success for date:%s", code_id, mdate) return (code_id, True) else: self.logger.error("%s set k data failed for date:%s", code_id, mdate) return (code_id, False) except Exception as e: self.logger.error("%s set k data for date %s exception:%s", code_id, mdate, e) return (code_id, False) #get stock bonus info bonus_info = pd.read_csv("/data/tdx/base/bonus.csv", sep=',', dtype={ 'code': str, 'market': int, 'type': int, 'money': float, 'price': float, 'count': float, 'rate': float, 'date': int }) index_info = CIndex('000001').get_k_data() if index_info is None or index_info.empty: return False df = self.stock_info_client.get() if df.empty: return False failed_list = df.code.tolist() if cdate is None: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) return process_concurrent_run(cfunc, failed_list, num=8) else: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) succeed = True if not process_concurrent_run(cfunc, failed_list, num=8): succeed = False return succeed #start_date = get_day_nday_ago(cdate, num = 4, dformat = "%Y-%m-%d") #for mdate in get_dates_array(start_date, cdate, asending = True): # if self.cal_client.is_trading_day(mdate): # self.logger.info("start recording stock info: %s", mdate) # cfunc = partial(_set_stock_info, mdate, bonus_info, index_info) # if not process_concurrent_run(cfunc, failed_list, num = 500): # self.logger.error("compute stock info for %s failed", mdate) # return False #return True def init_industry_info(self, cdate, num=1): def _set_industry_info(cdate, code_id): return (code_id, CIndex(code_id).set_k_data(cdate)) df = self.industry_info_client.get() if cdate is None: cfunc = partial(_set_industry_info, cdate) return concurrent_run(cfunc, df.code.tolist(), num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_industry_info, mdate) if not concurrent_run(cfunc, df.code.tolist(), num=5): succeed = False return succeed def init_yesterday_hk_info(self, cdate, num): succeed = True for data in ((ct.SH_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL), (ct.SZ_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL)): if not self.connect_client.set_market(data[0], data[1]): self.logger.error("connect_client for %s failed" % data) succeed = False continue if not self.connect_client.update(cdate, num=num): succeed = False self.connect_client.close() self.connect_client.quit() return succeed def get_concerned_index_codes(self): index_codes = list(ct.INDEX_DICT.keys()) #添加MSCI板块 index_codes.append('880883') return index_codes def init_index_components_info(self, cdate=None): if cdate is None: cdate = datetime.now().strftime('%Y-%m-%d') def _set_index_info(code_id): if code_id in self.index_objs: _obj = self.index_objs[code_id] else: _obj = CIndex(code_id) if code_id in list( ct.INDEX_DICT.keys()) else TdxFgIndex(code_id) return (code_id, _obj.set_components_data(cdate)) index_codes = self.get_concerned_index_codes() return concurrent_run(_set_index_info, index_codes, num=10) def set_bull_stock_ratio(self, cdate, num=10): def _set_bull_stock_ratio(code_id): return (code_id, BullStockRatio(code_id).update(cdate, num)) index_codes = self.get_concerned_index_codes() return concurrent_run(_set_bull_stock_ratio, index_codes) def init_tdx_index_info(self, cdate=None, num=1): def _set_index_info(cdate, code_id): try: if code_id in self.index_objs: _obj = self.index_objs[code_id] else: _obj = CIndex(code_id) if code_id in list( ct.TDX_INDEX_DICT.keys()) else TdxFgIndex(code_id) return (code_id, _obj.set_k_data(cdate)) except Exception as e: self.logger.error(e) return (code_id, False) #index_code_list = self.get_concerned_index_codes() index_code_list = list(ct.TDX_INDEX_DICT.keys()) if cdate is None: cfunc = partial(_set_index_info, cdate) return concurrent_run(cfunc, index_code_list, num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=num, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_index_info, mdate) if not concurrent_run(cfunc, index_code_list, num=5): succeed = False return succeed
class DataManager: def __init__(self, dbinfo=ct.DB_INFO, redis_host=None): self.dbinfo = dbinfo self.logger = getLogger(__name__) self.index_objs = dict() self.stock_objs = dict() self.combination_objs = dict() self.cal_client = CCalendar(dbinfo, redis_host) self.index_info_client = IndexInfo() self.comb_info_client = CombinationInfo(dbinfo, redis_host) self.stock_info_client = CStockInfo(dbinfo, redis_host) self.rindex_stock_data_client = RIndexStock(dbinfo, redis_host) self.industry_info_client = IndustryInfo(dbinfo, redis_host) self.rindustry_info_client = RIndexIndustryInfo(dbinfo, redis_host) self.limit_client = CLimit(dbinfo, redis_host) self.animation_client = CAnimation(dbinfo, redis_host) self.subscriber = Subscriber() self.quote_handler = StockQuoteHandler() self.ticker_handler = TickerHandler() self.connect_client = StockConnect(market_from=ct.SH_MARKET_SYMBOL, market_to=ct.HK_MARKET_SYMBOL, dbinfo=dbinfo, redis_host=redis_host) self.margin_client = Margin(dbinfo=dbinfo, redis_host=redis_host) self.emotion_client = Emotion(dbinfo=dbinfo, redis_host=redis_host) self.sh_exchange_client = StockExchange(ct.SH_MARKET_SYMBOL) self.sz_exchange_client = StockExchange(ct.SZ_MARKET_SYMBOL) def is_collecting_time(self, now_time=datetime.now()): _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] aft_open_hour, aft_open_minute, aft_open_second = (19, 00, 00) aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute, aft_open_second) aft_close_hour, aft_close_minute, aft_close_second = (23, 59, 59) aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute, aft_close_second) return aft_open_time < now_time < aft_close_time def is_morning_time(self, now_time=datetime.now()): _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] mor_open_hour, mor_open_minute, mor_open_second = (0, 0, 0) mor_open_time = datetime(y, m, d, mor_open_hour, mor_open_minute, mor_open_second) mor_close_hour, mor_close_minute, mor_close_second = (6, 30, 0) mor_close_time = datetime(y, m, d, mor_close_hour, mor_close_minute, mor_close_second) return mor_open_time < now_time < mor_close_time def collect_combination_runtime_data(self): def _combination_run(code_id): self.combination_objs[code_id].run() return (code_id, True) todo_iplist = list(self.combination_objs.keys()) return concurrent_run(_combination_run, todo_iplist, num=10) def collect_stock_runtime_data(self): if self.ticker_handler.empty(): return datas = self.ticker_handler.getQueue() while not datas.empty(): df = datas.get() df = df.set_index('time') df.index = pd.to_datetime(df.index) for code_str in set(df.code): code_id = code_str.split('.')[1] self.stock_objs[code_id].run(df.loc[df.code == code_str]) def init_real_stock_info(self): concerned_list = self.comb_info_client.get_concerned_list() prefix_concerned_list = [add_prifix(code) for code in concerned_list] ret = self.subscriber.subscribe(prefix_concerned_list, SubType.TICKER, self.ticker_handler) if 0 == ret: for code in concerned_list: if code not in self.stock_objs: self.stock_objs[code] = CStock(code, self.dbinfo, should_create_influxdb=True, should_create_mysqldb=True) return ret def init_index_info(self): index_list = ct.INDEX_DICT.keys() prefix_index_list = [add_index_prefix(code) for code in index_list] ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE, self.quote_handler) if 0 != ret: self.logger.error("subscribe for index list failed") return ret for code in index_list: if code not in self.index_objs: self.index_objs[code] = CIndex(code, should_create_influxdb=True, should_create_mysqldb=True) def collect_index_runtime_data(self): if self.quote_handler.empty(): return datas = self.quote_handler.getQueue() while not datas.empty(): df = datas.get() df['time'] = df.data_date + ' ' + df.data_time df = df.drop(['data_date', 'data_time'], axis=1) df = df.set_index('time') df.index = pd.to_datetime(df.index) for code_str in set(df.code): code_id = code_str.split('.')[1] self.index_objs[code_id].run(df.loc[df.code == code_str]) def run(self, sleep_time): while True: try: if self.cal_client.is_trading_day(): if is_trading_time(): sleep_time = 1 if not self.subscriber.status(): self.subscriber.start() if 0 == self.init_index_info( ) and 0 == self.init_real_stock_info(): self.init_combination_info() else: self.logger.debug("enter stop dict time") self.subscriber.stop() else: self.collect_stock_runtime_data() self.collect_combination_runtime_data() self.collect_index_runtime_data() self.animation_client.collect() else: sleep_time = 60 if self.subscriber.status(): self.subscriber.stop() except Exception as e: traceback.print_exc() self.logger.error(e) time.sleep(sleep_time) def set_update_info(self, step_length, exec_date, cdate=None, filename=ct.STEPFILE): step_info = dict() if cdate is None: cdate = 'none' step_info[cdate] = dict() step_info[cdate]['step'] = step_length step_info[cdate]['date'] = exec_date with open(filename, 'w') as f: json.dump(step_info, f) self.logger.info("finish step :%s" % step_length) def get_update_info(self, cdate=None, exec_date=None, filename=ct.STEPFILE): if cdate is None: cdate = 'none' if not os.path.exists(filename): return (0, exec_date) with open(filename, 'r') as f: step_info = json.load(f) if cdate not in step_info: return (0, exec_date) return (step_info[cdate]['step'], step_info[cdate]['date']) def bootstrap(self, cdate=None, exec_date=datetime.now().strftime('%Y-%m-%d')): finished_step, exec_date = self.get_update_info(cdate, exec_date) self.logger.info("enter updating.%s" % finished_step) if finished_step < 1: if not self.cal_client.init(): self.logger.error("cal_client init failed") return False self.set_update_info(1, exec_date, cdate) if finished_step < 2: if not self.index_info_client.update(): self.logger.error("index_info init failed") return False self.set_update_info(2, exec_date, cdate) if finished_step < 3: if not self.stock_info_client.update(): self.logger.error("stock_info init failed") return False self.set_update_info(3, exec_date, cdate) if finished_step < 4: if not self.comb_info_client.update(): self.logger.error("comb_info init failed") return False self.set_update_info(4, exec_date, cdate) if finished_step < 5: if not self.industry_info_client.update(): self.logger.error("industry_info init failed") return False self.set_update_info(5, exec_date, cdate) if finished_step < 6: if not self.download_and_extract(exec_date): self.logger.error("download_and_extract failed") return False self.set_update_info(6, exec_date, cdate) if finished_step < 7: if not self.init_tdx_index_info(cdate): self.logger.error("init_tdx_index_info failed") return False self.set_update_info(7, exec_date, cdate) if finished_step < 8: if not self.sh_exchange_client.update(exec_date, num=30): self.logger.error("sh exchange update failed") return False self.set_update_info(8, exec_date, cdate) if finished_step < 9: if not self.sz_exchange_client.update(exec_date, num=30): self.logger.error("sz exchange update failed") return False self.set_update_info(9, exec_date, cdate) if finished_step < 10: if not self.init_index_components_info(exec_date): self.logger.error("init index components info failed") return False self.set_update_info(10, exec_date, cdate) if finished_step < 11: if not self.init_industry_info(cdate): self.logger.error("init_industry_info failed") return False self.set_update_info(11, exec_date, cdate) if finished_step < 12: if not self.rindustry_info_client.update(exec_date): self.logger.error("init %s rindustry info failed" % exec_date) return False self.set_update_info(12, exec_date, cdate) if finished_step < 13: if not self.limit_client.update(exec_date): self.logger.error("init_limit_info failed") return False self.set_update_info(13, exec_date, cdate) if finished_step < 14: if not self.init_yesterday_hk_info(exec_date): self.logger.error("init_yesterday_hk_info failed") return False self.set_update_info(14, exec_date, cdate) if finished_step < 15: if not self.margin_client.update(exec_date): self.logger.error("init_yesterday_margin failed") return False self.set_update_info(15, exec_date, cdate) if finished_step < 16: if not self.init_stock_info(cdate): self.logger.error("init_stock_info set failed") return False self.set_update_info(16, exec_date, cdate) if finished_step < 17: if not self.init_base_float_profit(): self.logger.error("init base float profit for all stock") return False self.set_update_info(17, exec_date, cdate) if finished_step < 18: if not self.rindex_stock_data_client.update(exec_date, num=300): self.logger.error("rindex_stock_data set failed") return False self.set_update_info(18, exec_date, cdate) self.logger.info("updating succeed") return True def update(self, sleep_time): while True: self.logger.info("enter daily update process. %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) try: if self.cal_client.is_trading_day(): self.logger.info( "is trading day. %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) if self.is_collecting_time(): self.logger.info( "is collecting time. %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S')) self.bootstrap( cdate=datetime.now().strftime('%Y-%m-%d')) except Exception as e: kill_process("google-chrome") kill_process("renderer") kill_process("Xvfb") kill_process("zygote") kill_process("defunct") kill_process("show-component-extension-options") self.logger.error(e) time.sleep(sleep_time) def init_combination_info(self): trading_info = self.comb_info_client.get() for _, code_id in trading_info['code'].iteritems(): if str(code_id) not in self.combination_objs: self.combination_objs[str(code_id)] = Combination( code_id, self.dbinfo) def init_base_float_profit(self): def _set_base_float_profit(code_id): return (code_id, True) if CStock(code_id).set_base_floating_profit() else ( code_id, False) failed_list = self.stock_info_client.get().code.tolist() return process_concurrent_run(_set_base_float_profit, failed_list, num=500) def init_stock_info(self, cdate=None): def _set_stock_info(_date, bonus_info, index_info, code_id): try: if CStock(code_id).set_k_data(bonus_info, index_info, _date): self.logger.info("%s set k data success" % code_id) return (code_id, True) else: self.logger.error("%s set k data failed" % code_id) return (code_id, False) except Exception as e: self.logger.error("%s set k data exception:%s" % (code_id, e)) return (code_id, False) #get stock bonus info bonus_info = pd.read_csv("/data/tdx/base/bonus.csv", sep=',', dtype={ 'code': str, 'market': int, 'type': int, 'money': float, 'price': float, 'count': float, 'rate': float, 'date': int }) index_info = CIndex('000001').get_k_data() if index_info is None or index_info.empty: return False df = self.stock_info_client.get() failed_list = df.code.tolist() if cdate is None: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) return process_concurrent_run(cfunc, failed_list, num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=10, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_stock_info, mdate, bonus_info, index_info) if not process_concurrent_run(cfunc, failed_list, num=500): succeed = False return succeed def init_industry_info(self, cdate): def _set_industry_info(cdate, code_id): return (code_id, CIndex(code_id).set_k_data(cdate)) df = self.industry_info_client.get() if cdate is None: cfunc = partial(_set_industry_info, cdate) return concurrent_run(cfunc, df.code.tolist(), num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=30, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_industry_info, mdate) if not concurrent_run(cfunc, df.code.tolist(), num=5): succeed = False return succeed def init_yesterday_hk_info(self, cdate): succeed = True for data in ((ct.SH_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL), (ct.SZ_MARKET_SYMBOL, ct.HK_MARKET_SYMBOL)): if not self.connect_client.set_market(data[0], data[1]): self.logger.error("connect_client for %s failed" % data) succeed = False continue if not self.connect_client.update(cdate): succeed = False self.connect_client.close() self.connect_client.quit() kill_process("zygote") kill_process("defunct") kill_process("show-component-extension-options") return succeed def init_index_components_info(self, cdate=None): if cdate is None: cdate = datetime.now().strftime('%Y-%m-%d') def _set_index_info(code_id): _obj = self.index_objs[ code_id] if code_id in self.index_objs else CIndex(code_id) return (code_id, _obj.set_components_data(cdate)) return concurrent_run(_set_index_info, list(ct.INDEX_DICT.keys()), num=10) def init_tdx_index_info(self, cdate=None): def _set_index_info(cdate, code_id): try: _obj = self.index_objs[ code_id] if code_id in self.index_objs else CIndex(code_id) return (code_id, _obj.set_k_data(cdate)) except Exception as e: self.logger.error(e) return (code_id, False) if cdate is None: cfunc = partial(_set_index_info, cdate) return concurrent_run(cfunc, list(ct.TDX_INDEX_DICT.keys()), num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=30, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_index_info, mdate) if not concurrent_run( cfunc, list(ct.TDX_INDEX_DICT.keys()), num=5): succeed = False return succeed def download_and_extract(self, cdate): try: if not download(ct.ZIP_DIR, cdate): return False list_files = os.listdir(ct.ZIP_DIR) for filename in list_files: if not filename.startswith('.'): file_path = os.path.join(ct.ZIP_DIR, filename) if os.path.exists(file_path): unzip(file_path, ct.TIC_DIR) return True except Exception as e: self.logger.error(e) return False
from ccalendar import CCalendar from cstock_info import CStockInfo from base.cdate import get_day_nday_ago, get_dates_array if __name__ == '__main__': num = 5500 end_date = '2019-08-13' stock_info_client = CStockInfo() df = stock_info_client.get() code_list = df['code'].tolist() name_list = df['name'].tolist() code2namedict = dict(zip(code_list, name_list)) start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") date_array = get_dates_array(start_date, end_date) auth('18701683341', '52448oo78') for code in [ '000001', '000016', '000300', '000905', '399001', '399005', '399673' ]: obj = CIndex(code) for mdate in date_array: if CCalendar.is_trading_day(mdate): table_name = obj.get_components_table_name(mdate) if obj.is_table_exists(table_name): obj.mysql_client.delete(table_name) for mdate in date_array: if CCalendar.is_trading_day(mdate): if not obj.set_components_data_from_joinquant( code2namedict, mdate): print("{} for {} set failed".format(code, mdate))
class DataPreparer: def __init__(self): self.logger = getLogger(__name__) self.cal_client = CCalendar( dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1', filepath='/Volumes/data/quant/stock/conf/calAll.csv') def is_collecting_time(self): now_time = datetime.now() _date = now_time.strftime('%Y-%m-%d') y, m, d = time.strptime(_date, "%Y-%m-%d")[0:3] aft_open_hour, aft_open_minute, aft_open_second = (16, 00, 00) aft_open_time = datetime(y, m, d, aft_open_hour, aft_open_minute, aft_open_second) aft_close_hour, aft_close_minute, aft_close_second = (22, 00, 00) aft_close_time = datetime(y, m, d, aft_close_hour, aft_close_minute, aft_close_second) #self.logger.info("collecting now time. open_time:%s < now_time:%s < close_time:%s" % (aft_open_time, now_time, aft_close_time)) return aft_open_time < now_time < aft_close_time def run(self, cmd, timeout): self.logger.info("start to run cmd:%s, timeout:%s" % (cmd, timeout)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) finished = False for t in range(timeout): if not finished: time.sleep(2) if proc.poll() is not None: outs, errs = proc.communicate() self.logger.debug( "stdout:%s, stderr:%s" % (outs.decode("utf-8"), errs.decode("utf-8"))) finished = True else: for stdout_line in proc.stdout: self.logger.debug("stdout:%s" % stdout_line.decode("utf-8")) else: if proc.poll() is None: self.logger.error( "kill process after finished, cmd:%s, pid:%s" % (cmd, proc.pid)) proc.kill() return self.logger.error("kill process for not finished, cmd:%s, pid:%s" % (cmd, proc.pid)) proc.kill() def update(self, sleep_time): while True: try: self.logger.debug("enter update") if self.cal_client.is_trading_day(redis=self.cal_client.redis): if self.is_collecting_time(): ndate = get_latest_data_date( filepath= "/Volumes/data/quant/stock/data/stockdatainfo.json" ) mdate = transfer_date_string_to_int( datetime.now().strftime('%Y-%m-%d')) if ndate < mdate: self.run(SCRIPT1, timeout=600) self.run(SCRIPT2, timeout=2400) except Exception as e: self.logger.error(e) time.sleep(sleep_time)