def read_data_and_calc_indicators(self, code, start, end):
if self.freq.is_minutes():
data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
code, start, end, freq=self.freq, typ=SeType.Stock)
else:
data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single(
code, start, end)
if data is None or data.shape[0] < 10:
return
data = self.indicator_func(data)
if data is None or len(data) < 1:
return None
else:
return self.pre_save(data)
def re_calc_qfq_and_overwrite(code, start, end):
start = MIN_START_DATE
end = date_time_utils.Date().as_str()
stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
code, start, end)
qfq_data = stock_to_qfq(stock_data)
DB_CLIENT.replace_data_by_codes(qfq_data, DAY_QFQ_REPO)
def recreate_stock_qfq_data():
start = MIN_START_DATE
end = date_time_utils.Date().as_str()
init_qfq_repo()
print('START recreate QFQ data')
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
execute_tasks(code_list, update_qfq_for_single_code, start, end)
print('DONE recreate QFQ data')
def stock_to_qfq(data, verbose=False):
start = str(data.index.levels[0].min())[0:10]
end = str(data.index.levels[0].max())[0:10]
code = data.index.levels[1][0]
xdxr_data = SECURITY_DATA_READ_SERVICE.read_fq_info(code, start, end)
return do_prices_qfq(bfq_data=data, xdxr_data=xdxr_data, verbose=verbose)
def _read_bm_data(self, bm_code):
return SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
bm_code,
self.account.start_date,
self.account.end_date,
typ=SeType.Index
).reset_index().\
drop(columns=['code', 'open', 'high', 'low', 'vol', 'amount',
'up_count', 'down_count']).\
set_index('date')
def recreate_all_codes(self, start_date, end):
DB_CLIENT.remove_all_from(self.repo_name_for_backtest())
DB_CLIENT.remove_all_from(self.repo_name_for_trade())
DB_CLIENT.create_index_for_collection_with_date_and_code(self.repo_name_for_backtest())
DB_CLIENT.create_index_for_collection_with_date_and_code(self.repo_name_for_trade())
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
start = date_time_utils.Date.from_str(start_date)
end = date_time_utils.Date.from_str(end)
executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def update_qfq_for_single_code(code, start, end):
max_qfq_date = DB_CLIENT.read_max_date_for(code, DAY_QFQ_REPO)
if max_qfq_date is None:
calc_qfq_and_update(code, start, end)
else:
fq_info = SECURITY_DATA_READ_SERVICE.read_fq_info(code, start, end)
if fq_info is None:
copy_stock_day_directly(code, start, end)
else:
max_fq_info_date = fq_info.index.levels[0].max()
if str(max_fq_info_date)[0:10] < max_qfq_date:
copy_stock_day_directly(code, start, end)
else:
print("Recreate needed for {}, max qfq date={}, max fq info date={}".format(code, max_qfq_date, max_fq_info_date))
re_calc_qfq_and_overwrite(code, start, end)
def recreate_all_codes(self,
codes,
create_start_date=DEFAULT_MIN_START_DATE):
if codes is None:
if self.db_service.confirm_and_drop():
codes = SECURITY_DATA_READ_SERVICE.read_security_codes()
else:
return
else:
self.db_service.remove_all(codes)
end = Date()
start = Date.from_str(create_start_date)
executor.execute_tasks(codes, self.update_for_single_code,
start.as_str(), end.as_str())
def update_for_single_code(self, code, start, end):
data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single(code, start, end)
if data is not None and len(data) > 1:
data = dummy_appender.append_dummy_rows(data, 1)
data = self.indicator_func(data)
if data is None or data.shape[0] < 1:
pass
else:
df_without_dummy = dummy_appender.exclude_dummy_rows(data)
DB_CLIENT.update_data_append_newer(df_without_dummy, self.repo_name_for_backtest())
df_with_dummy_for_trade = data.tail(self.dummy_rows)
min_start_date = date_time_utils.Date.from_str(end). \
get_before(days=MasterIndicatorDB.MAX_CALENDAR_DAYS_TO_KEEP)
df_with_dummy_for_trade = df_with_dummy_for_trade.reset_index(level=0)
df_with_dummy_for_trade = df_with_dummy_for_trade.loc[df_with_dummy_for_trade.date > min_start_date.as_str()]
DB_CLIENT.replace_data_by_codes(df_with_dummy_for_trade, self.repo_name_for_trade())
def update_for_single_code(self, code, start, end):
data = SECURITY_DATA_READ_SERVICE.read_qfq_security_data_for_single(
code, start, end)
if data is None or data.shape[0] < 10:
return
master_data = self.master_service.read_master([code], start, end)
if master_data is None or len(master_data) < 1:
return
data = self.compute_indicator(data)
if data is None or len(data) < 1:
return
master_key = self.master_service.get_key()
data[master_key] = master_data[master_key]
data = data.loc[data[master_key] == True]
data.drop(columns=[master_key], inplace=True)
if data is not None and data.shape[0] > 0:
self.save(data)
def fetch_and_update_min_bars_by_csv():
dataService = CSV_SECURITY_DATA_SERVICE
monitor = SecurityDataUpdateMonitor()
codes = SECURITY_DATA_READ_SERVICE.read_security_codes()
# codes = ['000001']
a = Date()
start = '2018-01-01'
end = '2018-12-31'
# monitor.add_monitor_repo("stock_1min")
# monitor.add_monitor_repo("stock_5min")
# monitor.add_monitor_repo("stock_30min")
# monitor.collect_baseline()
# codes = ['000001', '300343'] + codes[:100]
dataService.update_bars_by_dates(start,
end,
typ=SeType.Stock,
freq=SeFreq.ONE_MIN,
codes=codes)
# dataService.refetch_and_save_bars(typ=SeType.Stock, freq=SeFreq.FIVE_MIN, codes=codes)
# dataService.update_bars(typ=SeType.Stock, freq=SeFreq.THIRTY_MIN, codes=None)
# monitor.collect_info_after_update(a)
b = Date()
print("time used: {} seconds".format(b.delta_to(a).seconds))
def update_all_codes(self, days_to_update=60):
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
end = date_time_utils.Date()
start = end.get_before(days=int(days_to_update * 1.5))
executor.execute_tasks(code_list, self.update_for_single_code, start.as_str(), end.as_str())
def update_stock_qfq_data():
end = date_time_utils.Date()
start = end.get_before(days=15)
code_list = SECURITY_DATA_READ_SERVICE.read_security_codes()
execute_tasks(code_list, update_qfq_for_single_code, start.as_str(),
end.as_str())
def calc_qfq_and_update(code, start, end):
stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
code, start, end)
if stock_data is not None and len(stock_data) > 0:
qfq_data = stock_to_qfq(stock_data)
DB_CLIENT.update_data_append_newer(qfq_data, DAY_QFQ_REPO)
def copy_stock_day_directly(code, start, end):
stock_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
code, start, end)
DB_CLIENT.update_data_append_newer(stock_data, DAY_QFQ_REPO)
def update_all_foot_peak_marks():
codes = SECURITY_DATA_READ_SERVICE.read_security_codes()
for fp_mark_service in DAY_FP_MARK_SERVICES:
fp_mark_service.update_all_codes(codes)
def _read_bm_data(self):
self.bm_data = SECURITY_DATA_READ_SERVICE.read_security_data_for_single(
self.bm_code,
self.account.start_date,
self.account.end_date,
typ=SeType.Index)
