def init_and_clause(self):
if self.period == PRD.INTRADAY:
minutes = self.aggregation * self.limit
days = int(minutes / (60 * 24)) + 1
dt_start = MyDate.get_date_from_datetime()
dt_start = MyDate.adjust_by_days(dt_start, -days)
dt_end = MyDate.get_date_from_datetime()
dt_end = MyDate.adjust_by_days(dt_end, 1)
self._and_clause = self.get_and_clause(dt_start, dt_end)
def __get_search_and_clause__(self):
date_from = MyDate.get_datetime_object(self.range_start_date_str)
date_to = MyDate.get_datetime_object(self.range_end_date_str)
date_from_adjusted = MyDate.adjust_by_days(date_from,
-self.range_length_days)
date_to_adjusted = MyDate.adjust_by_days(date_to,
self.range_length_days)
return "Date BETWEEN '{}' AND '{}'".format(date_from_adjusted,
date_to_adjusted)
def get_and_clause(dt_start=None, dt_end=None):
if dt_start is None:
date_start = MyDate.get_date_from_datetime()
date_start = MyDate.adjust_by_days(date_start, -1)
dt_end = MyDate.get_date_from_datetime()
dt_end = MyDate.adjust_by_days(dt_end, 1)
else:
date_start = MyDate.get_date_from_datetime(dt_start)
if dt_end:
date_end = MyDate.get_date_from_datetime(dt_end)
return "Date BETWEEN '{}' AND '{}'".format(date_start, date_end)
return "Date >= '{}'".format(date_start)
def get_trade_test_api_by_selected_trade_row(row, test_process: str) -> TradeTestApi:
api = TradeTestApi()
api.trade_id = row[DC.TRADE_ID] if DC.TRADE_ID in row else row[DC.ID]
api.pattern_id = row[DC.PATTERN_ID] if DC.PATTERN_ID in row else ''
api.test_process = test_process # e.g. TP.TRADE_REPLAY
api.pattern_type = row[DC.PATTERN_TYPE]
# api.buy_trigger = row[DC.BUY_TRIGGER]
api.trade_strategy = row[DC.TRADE_STRATEGY]
api.symbol = row[DC.TICKER_ID]
api.dt_start = MyDate.adjust_by_days(row[DC.PATTERN_RANGE_BEGIN_DT], -30)
api.dt_end = MyDate.adjust_by_days(row[DC.PATTERN_RANGE_END_DT], 30) # we need this correction for a smooth cont.
api.and_clause = PatternDataProvider.get_and_clause(api.dt_start, api.dt_end)
api.and_clause_unlimited = PatternDataProvider.get_and_clause(api.dt_start)
return api
def __get_graph__(self,
ticker_id: str,
refresh_interval: int,
limit: int = 0):
period = self.sys_config.period
aggregation = self.sys_config.period_aggregation
graph_cache_id = self.sys_config.graph_cache.get_cache_id(
ticker_id, period, aggregation, limit)
graph = self.sys_config.graph_cache.get_cached_object_by_key(
graph_cache_id)
if graph is not None:
return graph, graph_cache_id
if period == PRD.DAILY and self._recommender_table.selected_index != INDICES.CRYPTO_CCY:
self.sys_config.data_provider.from_db = True
else:
self.sys_config.data_provider.from_db = False
date_start = MyDate.adjust_by_days(MyDate.get_datetime_object().date(),
-limit)
and_clause = "Date > '{}'".format(date_start)
graph_title = self.sys_config.graph_cache.get_cache_title(
ticker_id, period, aggregation, limit)
detector = self._pattern_controller.get_detector_for_fibonacci_and_pattern(
self.sys_config, ticker_id, and_clause, limit)
graph_api = DccGraphApi(graph_cache_id, graph_title)
graph_api.ticker_id = ticker_id
graph_api.df = detector.pdh.pattern_data.df
graph = self.__get_dcc_graph_element__(detector, graph_api)
cache_api = self.sys_config.graph_cache.get_cache_object_api(
graph_cache_id, graph, period, refresh_interval)
self.sys_config.graph_cache.add_cache_object(cache_api)
return graph, graph_cache_id
def __get_date_range_for_index_calculation__(self) -> list:
date_range_values = self._access_layer.get_date_range_for_index(
self._index)
if self._df_index.shape[0] == 0:
return date_range_values
date_start = self._df_index.iloc[-1][CN.DATE]
date_start = MyDate.get_date_str_from_datetime(
MyDate.adjust_by_days(date_start, 1))
return [date_start, date_range_values[1]]
def update_equity_records(self) -> SalesmanDatabaseUpdateJobResult:
result_obj = SalesmanDatabaseUpdateJobResult()
access_layer = AccessLayer4Sale(self.db_salesman)
dt_today = MyDate.get_date_from_datetime()
# dt_today = MyDate.adjust_by_days(dt_today, 40)
dt_valid_until = MyDate.adjust_by_days(dt_today, 30)
dt_today_str = str(dt_today)
dt_valid_until_str = str(dt_valid_until)
return result_obj
def fill_test_data_for_pattern(tc: TradeTestCase, pattern: Pattern, tick_list_for_replay=None):
tc.symbol = pattern.ticker_id
from_date = MyDate.adjust_by_days(pattern.part_entry.date_first, -20)
if pattern.part_entry.breakout:
to_date = pattern.part_entry.breakout.tick_previous.date
else:
to_date = pattern.part_entry.pattern_range.tick_last.date
tc.and_clause = "Date BETWEEN '{}' AND '{}'".format(from_date, to_date)
tc.wave_tick_list = pattern.get_back_testing_wave_ticks(tick_list_for_replay)
def init_by_pattern_id_str(
self, pattern_id: str
): # example: 1_1_1_AAPL_12_2015-12-03_00:00_2016-01-07_00:00
pattern_id_obj = PatternID(**{'pattern_id': pattern_id})
symbol = pattern_id_obj.ticker_id
self.data_provider.from_db = True
self.data_provider.period = pattern_id_obj.period
self.data_provider.aggregation = pattern_id_obj.aggregation
self.data_provider.use_own_dic({symbol: symbol})
self.config.pattern_type_list = [pattern_id_obj.pattern_type]
range_adjusting = max(60, pattern_id_obj.range_length)
date_from_adjusted = MyDate.adjust_by_days(pattern_id_obj.date_start,
-range_adjusting)
date_to_adjusted = MyDate.adjust_by_days(pattern_id_obj.date_end,
2 * range_adjusting)
self.data_provider.and_clause = "Date BETWEEN '{}' AND '{}'".format(
date_from_adjusted, date_to_adjusted)
self.runtime_config.actual_pattern_range_from_time_stamp = pattern_id_obj.ts_from
self.runtime_config.actual_pattern_range_to_time_stamp = pattern_id_obj.ts_to
def update_trade_records(self,
mean: int,
sma_number: int,
trade_strategies: dict = None,
pattern_end_date=str):
if pattern_end_date == '':
pattern_end_date = str(MyDate.adjust_by_days(
None, -90)) # only the ones which are 3 month back
self.sys_config.init_detection_process_for_automated_trade_update(
mean, sma_number)
if trade_strategies is None:
trade_strategies = {
BT.BREAKOUT: [
TSTR.LIMIT, TSTR.LIMIT_FIX, TSTR.TRAILING_STOP,
TSTR.TRAILING_STEPPED_STOP
]
}
for pattern_type in FT.get_long_trade_able_types():
where_clause = "pattern_type = '{}' and period = 'DAILY' and trade_type in ('', 'long')".format(
pattern_type)
if pattern_end_date != '':
where_clause += " AND {} > '{}'".format(
DC.PATTERN_END_DT, pattern_end_date)
df_pattern_for_pattern_type = self.db_stock.get_pattern_records_as_dataframe(
where_clause)
pattern_id_dict = {
row[DC.ID]: row[DC.TRADE_TYPE]
for index, row in df_pattern_for_pattern_type.iterrows()
}
# pattern_id_list = ['20_1_1_LTCUSD_20_2016-11-02_00:00_2016-12-04_00:00']
counter = 0
for pattern_id, trade_type in pattern_id_dict.items():
counter += 1
run_detail = '{} ({:03d} of {:03d}): {}'.format(
pattern_type, counter, len(pattern_id_dict), pattern_id)
result_dict = self.db_stock.get_missing_trade_strategies_for_pattern_id(
pattern_id, trade_strategies, mean, sma_number)
for buy_trigger, trade_strategy_list in result_dict.items():
if len(trade_strategy_list) == 0:
print('{}: OK'.format(run_detail))
else:
print('{}: processing...'.format(run_detail))
self.sys_config.config.pattern_ids_to_find = [
pattern_id
]
self.sys_config.exchange_config.trade_strategy_dict = {
buy_trigger: trade_strategy_list
}
pattern_controller = PatternDetectionController(
self.sys_config)
pattern_controller.run_pattern_detector()
if trade_type == '':
self.db_stock.update_trade_type_for_pattern(
pattern_id, TRT.LONG)
def update_wave_records_for_daily_period(self, ticker_id: str, limit: int,
last_days: int):
self.sys_config.config.save_wave_data = True
self.sys_config.data_provider.period = PRD.DAILY
self.sys_config.data_provider.from_db = True
for k in range(0, last_days + 1):
date_end = MyDate.adjust_by_days(
MyDate.get_datetime_object().date(), -k)
date_start = MyDate.adjust_by_days(
MyDate.get_datetime_object().date(), -limit - k)
and_clause = "Date > '{}' AND Date <= '{}'".format(
date_start, date_end)
print('update_wave_records_for_daily_period: {} for {}'.format(
ticker_id, and_clause))
if self.sys_config.db_stock.is_symbol_loaded(
ticker_id, and_clause=and_clause):
detector = self.pattern_controller.get_detector_for_fibonacci(
self.sys_config, ticker_id, and_clause, limit)
detector.save_wave_data()
else:
print('No data available for {} and {}'.format(
ticker_id, and_clause))
def get_historical_data_by_client(self):
self.wrapper.init_historical_data_queue()
date_from = MyDate.adjust_by_days(MyDate.get_datetime_object(), -180)
date_from_str = MyDate.get_date_time_as_string_from_date_time(
date_from, '%Y%m%d %H:%M:%S')
print(date_from_str)
qqq = Contract()
qqq.symbol = 'MMEN'
qqq.secType = 'STK'
qqq.exchange = 'CSE'
qqq.currency = 'CAD'
self.reqHistoricalData(4001, qqq, date_from_str, "1 M", "1 day",
"MIDPOINT", 1, 1, False, [])
return self.wrapper.get_historical_data()
def process_optimize_log_files(self):
log_types_for_processing = self.__get_log_types_for_process_optimize_log_files__()
date_compare = MyDate.get_date_str_from_datetime(MyDate.adjust_by_days(None, -7))
for log_type in log_types_for_processing:
file_path = self.get_file_path_for_log_type(log_type)
line_to_keep_list = []
with open(file_path, 'r') as file:
for line in file.readlines():
log_line = FileLogLine(line)
if log_line.is_valid:
if log_line.date >= date_compare:
line_to_keep_list.append(line)
else:
print('{}: Line not valid in log file: {}'.format(file_path, line))
MyFile(file_path).replace_file_when_changed(line_to_keep_list)
def init_detection_process_for_automated_pattern_update(self):
date_now = MyDate.get_date_from_datetime()
date_from = MyDate.adjust_by_days(date_now, -180)
self.data_provider.and_clause = "Date BETWEEN '{}' AND '{}'".format(
str(date_from), str(date_now))
self.data_provider.from_db = True
self.data_provider.period = PRD.DAILY
self.config.pattern_type_list = FT.get_all()
self.config.detection_process = PDP.UPDATE_PATTERN_DATA
self.config.plot_data = False
self.config.with_trading = False
self.config.save_pattern_data = True
self.config.save_trade_data = False
self.config.save_wave_data = False
self.config.plot_only_pattern_with_fibonacci_waves = False
self.config.plot_min_max = False
self.config.plot_volume = False
self.config.length_for_local_min_max = 2
self.config.length_for_local_min_max_fibonacci = 1
self.config.bound_upper_value = CN.CLOSE
self.config.bound_lower_value = CN.CLOSE
def __add_calculated_graph_to_cache__(self, graph_cache_id: str,
ticker_id: str, period: str,
aggregation: int, limit: int,
indicator: str,
refresh_interval: int):
date_start = MyDate.adjust_by_days(MyDate.get_datetime_object().date(),
-limit)
and_clause = "Date > '{}'".format(date_start)
graph_title = self.sys_config.graph_cache.get_cache_title(
ticker_id, period, aggregation, limit)
detector = self._pattern_controller.get_detector_for_fibonacci_and_pattern(
self.sys_config, ticker_id, and_clause, limit)
graph_api = DccGraphApi(graph_cache_id, graph_title)
graph_api.ticker_id = ticker_id
graph_api.indicator = None if indicator == INDI.NONE else indicator
graph_api.df = detector.pdh.pattern_data.df
graph = self.__get_dcc_graph_element__(detector, graph_api)
cache_api = self.sys_config.graph_cache.get_cache_object_api(
graph_cache_id, graph, period, refresh_interval)
self.sys_config.graph_cache.add_cache_object(cache_api)
return graph
def update_equity_records(self) -> StockDatabaseUpdateJobResult:
result_obj = StockDatabaseUpdateJobResult()
access_layer = AccessLayer4Equity(self.db_stock)
dt_today = MyDate.get_date_from_datetime()
# dt_today = MyDate.adjust_by_days(dt_today, 40)
dt_valid_until = MyDate.adjust_by_days(dt_today, 30)
dt_today_str = str(dt_today)
dt_valid_until_str = str(dt_valid_until)
exchange_equity_type_dict = self.__get_equity_dict__()
result_obj.number_processed_records = len(exchange_equity_type_dict)
for exchange, equity_type in exchange_equity_type_dict.items():
value_dict = access_layer.get_index_dict(exchange)
if access_layer.are_any_records_available_for_date(
dt_today, exchange, equity_type):
result_obj.number_deleted_records += access_layer.delete_existing_equities(
equity_type, exchange)
sleep(2)
self.__update_equity_records_for_equity_type__(
access_layer, dt_today_str, dt_valid_until_str, exchange,
equity_type)
result_obj.number_saved_records += 1
return result_obj