def re_calc_qfq_and_overwrite(code, start, end):
start = MIN_START_DATE
end = date_time_utils.Date().as_str()
stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
code, start, end)
qfq_data = stock_to_qfq(stock_data)
DB_CLIENT.replace_data_by_codes(qfq_data, DAY_QFQ_REPO)
def read_data_by_codes_and_date(
self,
repo_name,
codes,
start, end=None,
extra_filter=None,
):
if end is None:
end = date_time_utils.Date().as_str()
else:
end = str(end)[0:10]
start = str(start)[0:10]
query = {
'code': {'$in': codes},
"date": {
"$lte": end,
"$gte": start
}
}
if extra_filter is not None:
query.update(extra_filter)
coll = self.client[repo_name]
cursor = coll.find(query, {"_id": 0, 'update_time': 0, 'date_stamp': 0}, batch_size=READ_BATCH_SIZE)
return self.construct_dataframe(cursor)
def recreate_stock_qfq_data():
start = MIN_START_DATE
end = date_time_utils.Date().as_str()
init_qfq_repo()
print('START recreate QFQ data')
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
execute_tasks(code_list, update_qfq_for_single_code, start, end)
print('DONE recreate QFQ data')
def update_all_codes(self, codes):
end = date_time_utils.Date()
start = end.get_before(days=self.days_to_update())
executor.execute_tasks(
codes,
self.update_for_single_code,
start.as_str(),
end.as_str()
)
def replace_data_by_codes(self, data, repo_name, precision=DEFAULT_PRECISION):
self.ensure_client()
if data is None or len(data) < 1:
return
data = data.round(precision).reset_index()
codes = data.code.unique().tolist()
coll = self.client[repo_name]
coll.delete_many({'code': {'$in': codes}})
if len(data) != 0:
data['update_time'] = date_time_utils.Date().as_str_with_time()
return coll.insert_many(to_json(data))
def update_bars(self,
typ: SeType = SeType.Stock,
freq: SeFreq = SeFreq.DAY,
codes=None,
days_to_update=DAYS_TO_UPDATE):
DB_CLIENT.create_index(typ.repo_name(freq), freq.keys_to_index())
if codes is None:
codes = self.read_security_codes(typ)
end = date_time_utils.Date()
start = end.get_before(days_to_update)
executor.execute_tasks(codes, self.do_fetch_and_save_bars, typ,
start.as_str(), end.as_str(), freq)
print("These codes failed to update: [" + ','.join(self.failed_codes) +
"]")
def update_data_append_newer(self, data, repo_name, precision=DEFAULT_PRECISION):
self.ensure_client()
if data is None or len(data) < 1:
return
data = data.round(precision).reset_index()
codes = data.code.unique()
keep_flag = 'newer_to_keep'
data[keep_flag] = False
for code in codes:
max_date = self.read_max_date_for(code, repo_name)
code_filter = data.code == code
if max_date is None:
data.loc[code_filter, keep_flag] = True
elif len(data.loc[code_filter & (data.date > max_date)]) > 0:
data.loc[code_filter & (data.date > max_date), keep_flag] = True
data = data.loc[data[keep_flag] == True]
if data is not None and len(data) > 0:
data['update_time'] = date_time_utils.Date().as_str_with_time()
data.drop(columns=[keep_flag], inplace=True)
self.client[repo_name].insert_many(to_json(data))
def do_fetch_and_save_bars(self, code, typ: SeType, start, end,
freq: SeFreq):
repo_name = typ.repo_name(freq)
end_date = date_time_utils.Date().as_str()
max_date_in_db = DB_CLIENT.read_max_date_for(code, repo_name)
if max_date_in_db is None:
start_date = MIN_START_TIME
else:
start_date = date_time_utils.Date.from_str(max_date_in_db).as_str()
try:
if start_date != end_date:
df = self.fetcher.fetch_bars(typ, code, start, end, freq)
if len(df) < 1:
self.failed_codes.append(code)
else:
df = self.remove_columns_for_min_data(df, freq)
DB_CLIENT.update_data_append_newer(df, repo_name)
except Exception as e:
print('Failed to fetch bars for {} {}, reason={}'.format(
code, freq, e))
self.failed_codes.append(str(code))
def refetch_and_save_bars(self,
typ: SeType,
freq: SeFreq = SeFreq.DAY,
codes=None):
if codes is None:
if self.__confirm_drop():
self.drop_bar_repo(typ, freq)
print('删除完成。')
codes = self.read_security_codes(typ)
else:
print("操作已取消。")
return
else:
DB_CLIENT.remove_all_from(typ.repo_name(freq),
query={'code': {
'$in': codes
}})
DB_CLIENT.create_index(typ.repo_name(freq), freq.keys_to_index())
end = date_time_utils.Date()
start = date_time_utils.Date.from_str(MIN_START_TIME)
executor.execute_tasks(codes, self.do_fetch_and_save_bars, typ,
start.as_str(), end.as_str(), freq)
def update_stock_qfq_data():
end = date_time_utils.Date()
start = end.get_before(days=15)
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
execute_tasks(code_list, update_qfq_for_single_code, start.as_str(),
end.as_str())
def update_all_codes(self, days_to_update=60):
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
end = date_time_utils.Date()
start = end.get_before(days=int(days_to_update * 1.5))
executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def _insert_data(self, data, repo_name):
coll = self.client[repo_name]
data['update_time'] = date_time_utils.Date().as_str_with_time()
return coll.insert_many(to_json(data))