def re_calc_qfq_and_overwrite(code, start, end): start = MIN_START_DATE end = date_time_utils.Date().as_str() stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single( code, start, end) qfq_data = stock_to_qfq(stock_data) DB_CLIENT.replace_data_by_codes(qfq_data, DAY_QFQ_REPO)
def read_data_by_codes_and_date( self, repo_name, codes, start, end=None, extra_filter=None, ): if end is None: end = date_time_utils.Date().as_str() else: end = str(end)[0:10] start = str(start)[0:10] query = { 'code': {'$in': codes}, "date": { "$lte": end, "$gte": start } } if extra_filter is not None: query.update(extra_filter) coll = self.client[repo_name] cursor = coll.find(query, {"_id": 0, 'update_time': 0, 'date_stamp': 0}, batch_size=READ_BATCH_SIZE) return self.construct_dataframe(cursor)
def recreate_stock_qfq_data(): start = MIN_START_DATE end = date_time_utils.Date().as_str() init_qfq_repo() print('START recreate QFQ data') code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() execute_tasks(code_list, update_qfq_for_single_code, start, end) print('DONE recreate QFQ data')
def update_all_codes(self, codes): end = date_time_utils.Date() start = end.get_before(days=self.days_to_update()) executor.execute_tasks( codes, self.update_for_single_code, start.as_str(), end.as_str() )
def replace_data_by_codes(self, data, repo_name, precision=DEFAULT_PRECISION): self.ensure_client() if data is None or len(data) < 1: return data = data.round(precision).reset_index() codes = data.code.unique().tolist() coll = self.client[repo_name] coll.delete_many({'code': {'$in': codes}}) if len(data) != 0: data['update_time'] = date_time_utils.Date().as_str_with_time() return coll.insert_many(to_json(data))
def update_bars(self, typ: SeType = SeType.Stock, freq: SeFreq = SeFreq.DAY, codes=None, days_to_update=DAYS_TO_UPDATE): DB_CLIENT.create_index(typ.repo_name(freq), freq.keys_to_index()) if codes is None: codes = self.read_security_codes(typ) end = date_time_utils.Date() start = end.get_before(days_to_update) executor.execute_tasks(codes, self.do_fetch_and_save_bars, typ, start.as_str(), end.as_str(), freq) print("These codes failed to update: [" + ','.join(self.failed_codes) + "]")
def update_data_append_newer(self, data, repo_name, precision=DEFAULT_PRECISION): self.ensure_client() if data is None or len(data) < 1: return data = data.round(precision).reset_index() codes = data.code.unique() keep_flag = 'newer_to_keep' data[keep_flag] = False for code in codes: max_date = self.read_max_date_for(code, repo_name) code_filter = data.code == code if max_date is None: data.loc[code_filter, keep_flag] = True elif len(data.loc[code_filter & (data.date > max_date)]) > 0: data.loc[code_filter & (data.date > max_date), keep_flag] = True data = data.loc[data[keep_flag] == True] if data is not None and len(data) > 0: data['update_time'] = date_time_utils.Date().as_str_with_time() data.drop(columns=[keep_flag], inplace=True) self.client[repo_name].insert_many(to_json(data))
def do_fetch_and_save_bars(self, code, typ: SeType, start, end, freq: SeFreq): repo_name = typ.repo_name(freq) end_date = date_time_utils.Date().as_str() max_date_in_db = DB_CLIENT.read_max_date_for(code, repo_name) if max_date_in_db is None: start_date = MIN_START_TIME else: start_date = date_time_utils.Date.from_str(max_date_in_db).as_str() try: if start_date != end_date: df = self.fetcher.fetch_bars(typ, code, start, end, freq) if len(df) < 1: self.failed_codes.append(code) else: df = self.remove_columns_for_min_data(df, freq) DB_CLIENT.update_data_append_newer(df, repo_name) except Exception as e: print('Failed to fetch bars for {} {}, reason={}'.format( code, freq, e)) self.failed_codes.append(str(code))
def refetch_and_save_bars(self, typ: SeType, freq: SeFreq = SeFreq.DAY, codes=None): if codes is None: if self.__confirm_drop(): self.drop_bar_repo(typ, freq) print('删除完成。') codes = self.read_security_codes(typ) else: print("操作已取消。") return else: DB_CLIENT.remove_all_from(typ.repo_name(freq), query={'code': { '$in': codes }}) DB_CLIENT.create_index(typ.repo_name(freq), freq.keys_to_index()) end = date_time_utils.Date() start = date_time_utils.Date.from_str(MIN_START_TIME) executor.execute_tasks(codes, self.do_fetch_and_save_bars, typ, start.as_str(), end.as_str(), freq)
def update_stock_qfq_data(): end = date_time_utils.Date() start = end.get_before(days=15) code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() execute_tasks(code_list, update_qfq_for_single_code, start.as_str(), end.as_str())
def update_all_codes(self, days_to_update=60): code_list = SECURITY_DATA_READ_SERVICE.read_security_codes() end = date_time_utils.Date() start = end.get_before(days=int(days_to_update * 1.5)) executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def _insert_data(self, data, repo_name): coll = self.client[repo_name] data['update_time'] = date_time_utils.Date().as_str_with_time() return coll.insert_many(to_json(data))