def collect_info_after_update(self, after: Date):
query_before = {'update_time': {'$lte': after.as_str_with_time()}}
query_after = {'update_time': {'$gt': after.as_str_with_time()}}
info_dict = self.info_after
for repo_name in self.repos_to_monitor:
cnt_before = DB_CLIENT.count_of(repo_name, query=query_before)
cnt_after = DB_CLIENT.count_of(repo_name, query=query_after)
info_dict[repo_name] = {
'cnt_before': cnt_before,
'cnt_after': cnt_after,
}
def recreate_all_codes(self,
codes,
create_start_date=DEFAULT_MIN_START_DATE):
if codes is None:
if self.db_service.confirm_and_drop():
codes = SECURITY_DATA_READ_SERVICE.read_security_codes()
else:
return
else:
self.db_service.remove_all(codes)
end = Date()
start = Date.from_str(create_start_date)
executor.execute_tasks(codes, self.update_for_single_code,
start.as_str(), end.as_str())
def fetch_and_update_min_bars_by_tdx():
dataService = TDX_SECURITY_DATA_UPDATE_SERVICE
monitor = SecurityDataUpdateMonitor()
codes = TDX_SECURITY_DATA_UPDATE_SERVICE.read_security_codes()
a = Date()
# monitor.add_monitor_repo("stock_1min")
# monitor.add_monitor_repo("stock_5min")
# monitor.add_monitor_repo("stock_30min")
# monitor.collect_baseline()
# codes = ['000001', '300343'] + codes[:100]
dataService.update_bars(typ=SeType.Stock, freq=SeFreq.ONE_MIN, codes=codes)
dataService.update_bars(typ=SeType.Stock, freq=SeFreq.FIVE_MIN, codes=codes)
dataService.update_bars(typ=SeType.Stock, freq=SeFreq.THIRTY_MIN, codes=None)
# monitor.collect_info_after_update(a)
b = Date()
print("time used: {} seconds".format(b.delta_to(a).seconds))
def fetch_and_update_by_tdx():
dataService = TDX_SECURITY_DATA_UPDATE_SERVICE
GCANDLE_CONFIG.set_mongodb_uri('mongodb://localhost:27017')
GCANDLE_CONFIG.set_mongodb_name('gcandle')
a = Date()
dataService.update_code_list()
dataService.update_bars(typ=SeType.Index,
freq=SeFreq.DAY,
days_to_update=500)
dataService.update_bars(typ=SeType.Stock,
freq=SeFreq.DAY,
days_to_update=500)
codes = dataService.read_security_codes(typ=SeType.Stock)
dataService.refetch_and_save_xdxr(codes)
# update_stock_qfq_data()
recreate_stock_qfq_data()
b = Date()
print("time used: {} seconds".format(b.delta_to(a).seconds))
def fetch_and_update_by_tdx():
dataService = TDX_SECURITY_DATA_UPDATE_SERVICE
monitor = SecurityDataUpdateMonitor()
a = Date()
monitor.add_monitor_repo("security_list")
monitor.add_monitor_repo("index_day")
monitor.add_monitor_repo("stock_day")
monitor.add_monitor_repo("stock_5min")
monitor.add_monitor_repo("stock_30min")
monitor.add_monitor_repo("security_xdxr")
monitor.collect_baseline()
dataService.update_code_list()
dataService.update_bars(typ=SeType.Index, freq=SeFreq.DAY)
dataService.update_bars(typ=SeType.Stock, freq=SeFreq.DAY)
codes = dataService.read_security_codes(typ=SeType.Stock)
dataService.refetch_and_save_xdxr(codes)
monitor.collect_info_after_update(a)
b = Date()
print("time used: {} seconds".format(b.delta_to(a).seconds))
monitor.report_changes()
def recreate_for_specified_codes(self,
codes,
start=DEFAULT_START_CREATE_DATE,
end=None):
if end is None:
end = Date().as_str()
self.repo.recreate_for_codes(codes, start, end)
print("{} master indicator recreate done".format(self.name))
for service in self.slave_services:
service.recreate_all_codes(codes, start, end)
print("{} slave indicator recreate done".format(
service.repo_name()))
def scan(self, argv):
if len(argv) > 1:
codes = self.master_service.read_all_codes()
end = argv[1]
start = Date(end).get_before(DAYS_BEFORE_TO_SCAN).as_str()
print('## Scan {} candidates with backtest data from {} to {}'.
format(self.name, start, end))
data = self.master_service.read_with_slave_for_backtest(codes,
start,
end=end)
else:
print('## Scan {} candidates for trade'.format(self.name))
data = self.master_service.read_with_slave_for_trade()
self.__do_scan(data)
def fetch_and_update_min_bars_by_csv():
dataService = CSV_SECURITY_DATA_SERVICE
monitor = SecurityDataUpdateMonitor()
codes = SECURITY_DATA_READ_SERVICE.read_security_codes()
# codes = ['000001']
a = Date()
start = '2018-01-01'
end = '2018-12-31'
# monitor.add_monitor_repo("stock_1min")
# monitor.add_monitor_repo("stock_5min")
# monitor.add_monitor_repo("stock_30min")
# monitor.collect_baseline()
# codes = ['000001', '300343'] + codes[:100]
dataService.update_bars_by_dates(start,
end,
typ=SeType.Stock,
freq=SeFreq.ONE_MIN,
codes=codes)
# dataService.refetch_and_save_bars(typ=SeType.Stock, freq=SeFreq.FIVE_MIN, codes=codes)
# dataService.update_bars(typ=SeType.Stock, freq=SeFreq.THIRTY_MIN, codes=None)
# monitor.collect_info_after_update(a)
b = Date()
print("time used: {} seconds".format(b.delta_to(a).seconds))
def recreate_for_all_codes(self,
start=DEFAULT_START_CREATE_DATE,
end=None):
if self.user_confirm_to_continue():
if end is None:
end = Date().as_str()
self.repo.recreate_all_codes(start, end)
print("{} master indicator recreate done".format(self.name))
codes = self.repo.read_all_codes()
for service in self.slave_services:
service.recreate_all_codes(codes, start, end)
print("{} slave indicator recreate done".format(
service.repo_name()))
else:
print("Cancelled")
def update_all_codes(self, codes):
end = Date()
start = end.get_before(self.days_to_update)
executor.execute_tasks(codes, self.update_for_single_code,
start.as_str(), end.as_str())
info_dict = self.info_after
for repo_name in self.repos_to_monitor:
cnt_before = DB_CLIENT.count_of(repo_name, query=query_before)
cnt_after = DB_CLIENT.count_of(repo_name, query=query_after)
info_dict[repo_name] = {
'cnt_before': cnt_before,
'cnt_after': cnt_after,
}
def report_changes(self):
if len(self.baseline) < 1 or len(self.info_after) < 1:
print("Invalid request, no baseline or info after change collected")
for k, v in self.baseline.items():
cnt_changed = self.info_after[k]
print('## Changes of {} ##'.format(k))
if cnt_changed is None or len(cnt_changed) < 1:
print('No info after collected for {}'.format(k))
continue
if v != cnt_changed['cnt_before']:
print('Warning: base cnt changed: {} -> {}'.format(v, cnt_changed['cnt_before']))
print('updated cnt: {}'.format(cnt_changed['cnt_after']))
if __name__ == '__main__':
dataService = SECURITY_DATA_READ_SERVICE
verifier = SecurityDataUpdateMonitor()
verifier.collect_baseline()
verifier.collect_info_after_update(after=Date())
verifier.report_changes()