def read_data_and_calc_indicators(self, code, start, end): if self.freq.is_minutes(): data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( code, start, end, freq=self.freq, typ=SeType.Stock) else: data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single( code, start, end) if data is None or data.shape[0] < 10: return data = self.indicator_func(data) if data is None or len(data) < 1: return None else: return self.pre_save(data)
def re_calc_qfq_and_overwrite(code, start, end): start = MIN_START_DATE end = date_time_utils.Date().as_str() stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( code, start, end) qfq_data = stock_to_qfq(stock_data) DB_CLIENT.replace_data_by_codes(qfq_data, DAY_QFQ_REPO)
def recreate_stock_qfq_data(): start = MIN_START_DATE end = date_time_utils.Date().as_str() init_qfq_repo() print('START recreate QFQ data') code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() execute_tasks(code_list, update_qfq_for_single_code, start, end) print('DONE recreate QFQ data')
def stock_to_qfq(data, verbose=False): start = str(data.index.levels[0].min())[0:10] end = str(data.index.levels[0].max())[0:10] code = data.index.levels[1][0] xdxr_data = SECURITY_DATA_READ_SERVICE.read_fq_info(code, start, end) return do_prices_qfq(bfq_data=data, xdxr_data=xdxr_data, verbose=verbose)
def _read_bm_data(self, bm_code): return SECURITY_DATA_READ_SERVICE.read_security_data_for_single( bm_code, self.account.start_date, self.account.end_date, typ=SeType.Index ).reset_index().\ drop(columns=['code', 'open', 'high', 'low', 'vol', 'amount', 'up_count', 'down_count']).\ set_index('date')
def recreate_all_codes(self, start_date, end): DB_CLIENT.remove_all_from(self.repo_name_for_backtest()) DB_CLIENT.remove_all_from(self.repo_name_for_trade()) DB_CLIENT.create_index_for_collection_with_date_and_code(self.repo_name_for_backtest()) DB_CLIENT.create_index_for_collection_with_date_and_code(self.repo_name_for_trade()) code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() start = date_time_utils.Date.from_str(start_date) end = date_time_utils.Date.from_str(end) executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def update_qfq_for_single_code(code, start, end): max_qfq_date = DB_CLIENT.read_max_date_for(code, DAY_QFQ_REPO) if max_qfq_date is None: calc_qfq_and_update(code, start, end) else: fq_info = SECURITY_DATA_READ_SERVICE.read_fq_info(code, start, end) if fq_info is None: copy_stock_day_directly(code, start, end) else: max_fq_info_date = fq_info.index.levels[0].max() if str(max_fq_info_date)[0:10] < max_qfq_date: copy_stock_day_directly(code, start, end) else: print("Recreate needed for {}, max qfq date={}, max fq info date={}".format(code, max_qfq_date, max_fq_info_date)) re_calc_qfq_and_overwrite(code, start, end)
def recreate_all_codes(self, codes, create_start_date=DEFAULT_MIN_START_DATE): if codes is None: if self.db_service.confirm_and_drop(): codes = SECURITY_DATA_READ_SERVICE.read_security_codes() else: return else: self.db_service.remove_all(codes) end = Date() start = Date.from_str(create_start_date) executor.execute_tasks(codes, self.update_for_single_code, start.as_str(), end.as_str())
def update_for_single_code(self, code, start, end): data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single(code, start, end) if data is not None and len(data) > 1: data = dummy_appender.append_dummy_rows(data, 1) data = self.indicator_func(data) if data is None or data.shape[0] < 1: pass else: df_without_dummy = dummy_appender.exclude_dummy_rows(data) DB_CLIENT.update_data_append_newer(df_without_dummy, self.repo_name_for_backtest()) df_with_dummy_for_trade = data.tail(self.dummy_rows) min_start_date = date_time_utils.Date.from_str(end). \ get_before(days=MasterIndicatorDB.MAX_CALENDAR_DAYS_TO_KEEP) df_with_dummy_for_trade = df_with_dummy_for_trade.reset_index(level=0) df_with_dummy_for_trade = df_with_dummy_for_trade.loc[df_with_dummy_for_trade.date > min_start_date.as_str()] DB_CLIENT.replace_data_by_codes(df_with_dummy_for_trade, self.repo_name_for_trade())
def update_for_single_code(self, code, start, end): data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single( code, start, end) if data is None or data.shape[0] < 10: return master_data = self.master_service.read_master([code], start, end) if master_data is None or len(master_data) < 1: return data = self.compute_indicator(data) if data is None or len(data) < 1: return master_key = self.master_service.get_key() data[master_key] = master_data[master_key] data = data.loc[data[master_key] == True] data.drop(columns=[master_key], inplace=True) if data is not None and data.shape[0] > 0: self.save(data)
def fetch_and_update_min_bars_by_csv(): dataService = CSV_SECURITY_DATA_SERVICE monitor = SecurityDataUpdateMonitor() codes = SECURITY_DATA_READ_SERVICE.read_security_codes() # codes = ['000001'] a = Date() start = '2018-01-01' end = '2018-12-31' # monitor.add_monitor_repo("stock_1min") # monitor.add_monitor_repo("stock_5min") # monitor.add_monitor_repo("stock_30min") # monitor.collect_baseline() # codes = ['000001', '300343'] + codes[:100] dataService.update_bars_by_dates(start, end, typ=SeType.Stock, freq=SeFreq.ONE_MIN, codes=codes) # dataService.refetch_and_save_bars(typ=SeType.Stock, freq=SeFreq.FIVE_MIN, codes=codes) # dataService.update_bars(typ=SeType.Stock, freq=SeFreq.THIRTY_MIN, codes=None) # monitor.collect_info_after_update(a) b = Date() print("time used: {} seconds".format(b.delta_to(a).seconds))
def update_all_codes(self, days_to_update=60): code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() end = date_time_utils.Date() start = end.get_before(days=int(days_to_update * 1.5)) executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def update_stock_qfq_data(): end = date_time_utils.Date() start = end.get_before(days=15) code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() execute_tasks(code_list, update_qfq_for_single_code, start.as_str(), end.as_str())
def calc_qfq_and_update(code, start, end): stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( code, start, end) if stock_data is not None and len(stock_data) > 0: qfq_data = stock_to_qfq(stock_data) DB_CLIENT.update_data_append_newer(qfq_data, DAY_QFQ_REPO)
def copy_stock_day_directly(code, start, end): stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( code, start, end) DB_CLIENT.update_data_append_newer(stock_data, DAY_QFQ_REPO)
def update_all_foot_peak_marks(): codes = SECURITY_DATA_READ_SERVICE.read_security_codes() for fp_mark_service in DAY_FP_MARK_SERVICES: fp_mark_service.update_all_codes(codes)
def _read_bm_data(self): self.bm_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( self.bm_code, self.account.start_date, self.account.end_date, typ=SeType.Index)