def __fill_possible_pattern_ranges__(self):
pattern_type_set = set(self.pattern_type_list)
if len(pattern_type_set.intersection(set(FT.get_normal_types()))) > 0:
self.range_detector_max = PatternRangeDetectorMax(self.sys_config)
self.range_detector_min = PatternRangeDetectorMin(self.sys_config)
if len(pattern_type_set.intersection(set(
FT.get_head_shoulder_types()))) > 0:
self.range_detector_h_s = PatternRangeDetectorHeadShoulder(
self.sys_config)
if len(
pattern_type_set.intersection(
set(FT.get_head_shoulder_bottom_types()))) > 0:
self.range_detector_h_s_b = PatternRangeDetectorHeadShoulderBottom(
self.sys_config)
if FT.FIBONACCI_ASC in self.pattern_type_list:
wave_list = [
wave for wave in self.fib_wave_tree.fibonacci_wave_list
if wave.wave_type == FD.ASC
]
self.range_detector_fib_asc = PatternRangeDetectorFibonacciAsc(
self.sys_config, wave_list)
if FT.FIBONACCI_DESC in self.pattern_type_list:
wave_list = [
wave for wave in self.fib_wave_tree.fibonacci_wave_list
if wave.wave_type == FD.DESC
]
self.range_detector_fib_desc = PatternRangeDetectorFibonacciDesc(
self.sys_config, wave_list)
def get_function_container_by_api(api: PatternFunctionContainerFactoryApi):
pattern_type = api.pattern_type
sys_config = api.sys_config
df = api.pattern_range.get_related_part_from_data_frame(api.df_min_max)
f_upper = api.get_f_upper()
f_lower = api.get_f_lower()
f_cont = PatternFunctionContainerFactory.get_function_container(
sys_config, pattern_type, df, f_lower, f_upper)
if FT.is_pattern_type_any_head_shoulder(api.pattern_type):
f_cont.set_tick_for_helper(
api.pattern_range.hsf.tick_shoulder_left)
elif FT.is_pattern_type_any_fibonacci(api.pattern_type):
f_cont.max_positions_after_tick_for_helper = api.pattern_range.fib_form.max_positions_after_tick_for_helper
return f_cont
def __init_by_pattern_id__(self, pattern_id_str: str):
# Example: 1_1_1_AAPL_12_2015-12-03_00:00_2016-01-07_00:00
parts = pattern_id_str.split('_')
self.equity_type_id = int(parts[0])
self.period_id = int(parts[1])
self.period = PRD.get_period(self.period_id)
self.aggregation = int(parts[2])
self.ticker_id = parts[3]
self.pattern_type_id = int(parts[4])
self.pattern_type = FT.get_pattern_type(self.pattern_type_id)
self.range_start_date_str = parts[5]
self.range_start_time_str = parts[6]
self.range_end_date_str = parts[7]
self.range_end_time_str = parts[8]
self.date_start = MyDate.get_datetime_object(self.range_start_date_str)
self.date_end = MyDate.get_datetime_object(self.range_end_date_str)
self.range_length = (self.date_end - self.date_start).days
self.range_id = '{}_{}_{}_{}'.format(self.range_start_date_str,
self.range_start_time_str,
self.range_end_date_str,
self.range_end_time_str)
self.ts_from = MyDate.get_epoch_seconds_from_datetime(parts[5])
self.ts_to = MyDate.get_epoch_seconds_from_datetime(parts[7])
self.range_length_days = int(
(self.ts_to - self.ts_from) / (60 * 60 * 24))
self.and_clause = self.__get_search_and_clause__()
def __get_pattern_color__(pattern: Pattern):
is_trade_able = FT.is_pattern_type_long_trade_able(
pattern.pattern_type)
if pattern.was_breakout_done():
return 'green' if is_trade_able else 'lightgreen'
else:
return 'yellow' if is_trade_able else 'khaki'
def init_detection_process_for_automated_trade_update(
self, mean: int, sma_number: int):
self.config.detection_process = PDP.UPDATE_TRADE_DATA
self.config.pattern_type_list = FT.get_long_trade_able_types()
self.exchange_config.trade_strategy_dict = {
BT.BREAKOUT: [
TSTR.LIMIT, TSTR.LIMIT_FIX, TSTR.TRAILING_STOP,
TSTR.TRAILING_STEPPED_STOP
]
}
self.exchange_config.default_trade_strategy_dict = {
BT.BREAKOUT: TSTR.LIMIT
}
self.config.detection_process = PDP.UPDATE_TRADE_DATA
self.config.plot_data = False
self.config.with_trading = True
self.config.trading_last_price_mean_aggregation = mean
self.config.simple_moving_average_number = sma_number # default = 8
self.config.save_pattern_data = False
self.config.save_trade_data = True
self.config.replace_existing_trade_data_on_db = False # default = False
self.config.plot_only_pattern_with_fibonacci_waves = False
self.config.plot_min_max = True
self.config.plot_volume = False
self.config.length_for_local_min_max = 2
self.config.length_for_local_min_max_fibonacci = 1
self.config.bound_upper_value = CN.CLOSE
self.config.bound_lower_value = CN.CLOSE
self.config.fibonacci_tolerance_pct = 0.1 # default is 0.20
self.sound_machine.is_active = False
def get_complementary_function_list(self, pattern_type: str) -> list:
if FT.is_pattern_type_any_head_shoulder(pattern_type):
return [self.f_param_parallel]
elif pattern_type in [FT.TKE_BOTTOM, FT.TKE_TOP]:
return [self.f_param_const]
else:
return [] if self._f_param_list is None else self._f_param_list
def __adjust_sys_config__(self):
print('TradeTest.__adjust_sys_config__')
self.sys_config.runtime_config.actual_trade_process = self.trade_process
self.sys_config.config.plot_data = False
if self.trade_process == TP.BACK_TESTING:
self.sys_config.config.pattern_type_list = FT.get_long_trade_able_types()
self.sys_config.config.save_pattern_data = False
self.sys_config.config.save_trade_data = False
def is_deletion_reason_candidate_for_black_buy_pattern_id_list(
pattern_trade: PatternTrade, deletion_reason: str):
if deletion_reason == PDR.PATTERN_VANISHED: # they have the trend to reappear
return False
elif deletion_reason == PDR.WRONG_BREAKOUT:
if pattern_trade.pattern.pattern_type in FT.get_fibonacci_types():
return False # they have the trend to reappear
return True
def __old_init__(self, equity_type_id: int, period: str, aggregation: int,
ticker_id: str, pattern_type: str, pattern_range_id: str):
self.equity_type_id = equity_type_id
self.period_id = PRD.get_id(period)
self.aggregation = aggregation
self.ticker_id = ticker_id
self.pattern_type_id = FT.get_id(pattern_type)
self.range_id = pattern_range_id
self.__init_by_pattern_id__(self.id)
def __get_optimal_pattern_type_dict_for_long_trading__(self):
opt_dict = {}
for pattern_type in FT.get_long_trade_able_types():
if self._pattern_type_pos_neg_result_dict[
pattern_type] >= self.expected_relation_pos_neg:
opt_dict[
pattern_type] = self._pattern_type_pos_neg_result_dict[
pattern_type]
return opt_dict
def __init__(self, **kwargs):
if 'pattern_id' in kwargs:
self.__init_by_pattern_id__(kwargs['pattern_id'])
else:
self.equity_type_id = kwargs['equity_type_id']
self.period_id = PRD.get_id(kwargs['_period'])
self.aggregation = kwargs['_aggregation']
self.ticker_id = kwargs['ticker_id']
self.pattern_type_id = FT.get_id(kwargs['pattern_type'])
self.range_id = kwargs['pattern_range_id']
self.__init_by_pattern_id__(self.id)
def update_trade_records(self,
mean: int,
sma_number: int,
trade_strategies: dict = None,
pattern_end_date=str):
if pattern_end_date == '':
pattern_end_date = str(MyDate.adjust_by_days(
None, -90)) # only the ones which are 3 month back
self.sys_config.init_detection_process_for_automated_trade_update(
mean, sma_number)
if trade_strategies is None:
trade_strategies = {
BT.BREAKOUT: [
TSTR.LIMIT, TSTR.LIMIT_FIX, TSTR.TRAILING_STOP,
TSTR.TRAILING_STEPPED_STOP
]
}
for pattern_type in FT.get_long_trade_able_types():
where_clause = "pattern_type = '{}' and period = 'DAILY' and trade_type in ('', 'long')".format(
pattern_type)
if pattern_end_date != '':
where_clause += " AND {} > '{}'".format(
DC.PATTERN_END_DT, pattern_end_date)
df_pattern_for_pattern_type = self.db_stock.get_pattern_records_as_dataframe(
where_clause)
pattern_id_dict = {
row[DC.ID]: row[DC.TRADE_TYPE]
for index, row in df_pattern_for_pattern_type.iterrows()
}
# pattern_id_list = ['20_1_1_LTCUSD_20_2016-11-02_00:00_2016-12-04_00:00']
counter = 0
for pattern_id, trade_type in pattern_id_dict.items():
counter += 1
run_detail = '{} ({:03d} of {:03d}): {}'.format(
pattern_type, counter, len(pattern_id_dict), pattern_id)
result_dict = self.db_stock.get_missing_trade_strategies_for_pattern_id(
pattern_id, trade_strategies, mean, sma_number)
for buy_trigger, trade_strategy_list in result_dict.items():
if len(trade_strategy_list) == 0:
print('{}: OK'.format(run_detail))
else:
print('{}: processing...'.format(run_detail))
self.sys_config.config.pattern_ids_to_find = [
pattern_id
]
self.sys_config.exchange_config.trade_strategy_dict = {
buy_trigger: trade_strategy_list
}
pattern_controller = PatternDetectionController(
self.sys_config)
pattern_controller.run_pattern_detector()
if trade_type == '':
self.db_stock.update_trade_type_for_pattern(
pattern_id, TRT.LONG)
def __adjust_stop_loss_to_pattern_type_specifics__(
self, last_price: float) -> bool:
module = '__adjust_stop_loss_to_pattern_type_specifics__'
if FT.is_pattern_type_any_triangle(self._pattern_type):
# expected breakout = high of the triangle, but stop loss at apex
apex_value = self._data_dict.get(DC.APEX_VALUE)
pattern_begin_high = self._data_dict.get(DC.PATTERN_BEGIN_HIGH)
max_apex_and_high = max(apex_value, pattern_begin_high)
if last_price > max_apex_and_high * 1.01 > max_apex_and_high > self._stop_loss: # we want to have a close stop_loss
self.__log_stop_loss_adjustment__(module, self._pattern_type,
max_apex_and_high)
self._stop_loss = max_apex_and_high
return True
return False
def get_detector_for_fibonacci_and_pattern(sys_config: SystemConfiguration,
ticker: str,
and_clause='',
limit=300) -> PatternDetector:
sys_config.init_pattern_data_handler_for_ticker_id(
ticker, and_clause, limit)
sys_config.config.pattern_type_list = FT.get_all()
sys_config.init_predictors_without_condition_list()
# sys_config.config.save_wave_data
print('\nProcessing for Fib & Pattern: {} ({})\n'.format(
ticker, sys_config.runtime_config.actual_ticker_name))
detector = PatternDetector(sys_config)
detector.parse_for_fibonacci_waves()
detector.add_prediction_data_to_wave()
detector.parse_for_pattern()
return detector
def __get_pattern_type_pos_neg_result_relation_as_dict__(self):
pattern_type_pos_neg_relation_dict = {}
for pattern_type in FT.get_all():
pattern_type_pos_neg_relation_dict[pattern_type] = 1 # default
df_pos = self.df_trades.loc[np.logical_and(
self.df_trades[DC.PATTERN_TYPE] == pattern_type,
self.df_trades[DC.TRADE_RESULT_PCT] > 1)]
df_neg = self.df_trades.loc[np.logical_and(
self.df_trades[DC.PATTERN_TYPE] == pattern_type,
self.df_trades[DC.TRADE_RESULT_PCT] < -1)]
if df_neg.shape[0] < 10: # not enough data
if df_pos.shape[0] > 20:
pattern_type_pos_neg_relation_dict[pattern_type] = 3
else:
pattern_type_pos_neg_relation_dict[pattern_type] = round(
df_pos.shape[0] / df_neg.shape[0], 1)
return pattern_type_pos_neg_relation_dict
def __get_drop_down_value_dict__(self) -> dict:
return {
DDT.CHART_TYPE:
CHT.get_chart_types_for_pattern_statistics(),
DDT.PREDICTOR:
PRED.get_for_pattern_all(),
DDT.CATEGORY:
PatternTable.get_columns_for_statistics_category(),
DDT.X_VARIABLE:
PatternTable.get_columns_for_statistics_x_variable(),
DDT.Y_VARIABLE:
PatternTable.get_columns_for_statistics_y_variable(),
DDT.CHART_TEXT_VARIABLE:
PatternTable.get_columns_for_statistics_text_variable(),
DDT.PATTERN_TYPE:
FT.get_all_for_statistics()
}
def set_trade_id_as_pattern_id_to_find(self, trade_id: str):
# From: Breakout-Expected_win-Limit_fix-mean04_DAILY_PG_Channel_2017-10-25_00:00_2017-11-07_00:00
# To: 1_1_1_KO_22_2016-07-22_00:00_2016-09-28_00:00
# return '{}-{}-{}-{}_{}'.format(
# self.buy_trigger, self.trade_box_type, self.trade_strategy, mean_aggregation, self.pattern.id_readable)
# def __get_pattern_id__(self) -> PatternID:
# kwargs = {
# 'equity_type_id': self.data_dict_obj.get(DC.EQUITY_TYPE_ID),
# '_period': self.data_dict_obj.get(DC.PERIOD),
# '_aggregation': self.data_dict_obj.get(DC.PROCESS),
# 'ticker_id': self.ticker_id,
# 'pattern_type': self.pattern_type,
# 'pattern_range_id': self.pattern_range.id
# }
# return PatternID(**kwargs)
# [0]: Breakout - Expected_
# [1]: win - Limit_
# [2]: fix - mean04_
# [3]: DAILY_
# [4]: PG_
# [5]: Channel_
# [6]: 2017 - 10 - 25 # _
# [7]: 00:00 # _
# [8]: 2017 - 11 - 07 # _
# [9]: 00:00
#
# Breakout - Expected_win - Trailing_stepped_stop - mean04_DAILY_LTCUSD_Channel_2016 - 12 - 0
# 9_00: 00_2016 - 12 - 17_00:00
# '10-10-30-DAILY-0-Channel-2016-12-09-00:00-2016-12-17'
trade_id_parts_01 = trade_id.split('-')
trade_id_parts_02 = trade_id.split('_')
buy_trigger_id = BT.get_id(trade_id_parts_01[0])
trade_box_type_id = BTT.get_id(trade_id_parts_01[1])
trade_strategy_id = TSTR.get_id(trade_id_parts_01[2])
symbol = trade_id_parts_02[4]
pattern_type_id = FT.get_id(trade_id_parts_02[5])
date_from = trade_id_parts_02[6]
time_from = trade_id_parts_02[7]
date_to = trade_id_parts_02[8]
time_to = trade_id_parts_02[9]
pattern_id = '-'.join([str(buy_trigger_id), str(trade_box_type_id), str(trade_strategy_id),
symbol, str(pattern_type_id),
date_from, time_from, date_to, time_to])
self.pattern_ids_to_find = [pattern_id]
def __get_drop_down_value_dict__(self) -> dict:
return {
DDT.CHART_TYPE:
CHT.get_chart_types_for_models_statistics(),
DDT.CATEGORY:
STBL.get_for_model_statistics(),
DDT.PREDICTOR:
PRED.get_for_pattern_all(),
DDT.X_VARIABLE:
PatternTable.get_label_columns_touch_points_for_statistics(),
DDT.Y_VARIABLE:
MTC.get_all_for_statistics(),
DDT.CHART_TEXT_VARIABLE:
PatternTable.get_columns_for_statistics_text_variable(),
DDT.MODEL_TYPE:
MT.get_all_for_statistics(),
DDT.PATTERN_TYPE:
FT.get_all_for_statistics()
}
def __check_for_loop_break__(pattern: Pattern, counter: int,
number_of_positions: int,
tick: WaveTick) -> bool:
if pattern.breakout is not None:
return True
if counter > number_of_positions: # maximal number for the whole pattern (from start)
return True
if pattern.function_cont.is_tick_breakout_on_wrong_side(tick):
return True
if counter > pattern.breakout_required_after_ticks:
return True
if tick.f_var > pattern.function_cont.f_var_cross_f_upper_f_lower > 0:
if not FT.is_pattern_type_any_fibonacci(pattern.pattern_type):
return True
# if not function_cont.is_regression_value_in_pattern_for_f_var(tick.f_var - 6):
# # check the last value !!! in some IMPORTANT cases is the breakout just on that day...
# return True
if not pattern.function_cont.is_tick_inside_pattern_range(tick):
return True
return False
def init_detection_process_for_automated_pattern_update(self):
date_now = MyDate.get_date_from_datetime()
date_from = MyDate.adjust_by_days(date_now, -180)
self.data_provider.and_clause = "Date BETWEEN '{}' AND '{}'".format(
str(date_from), str(date_now))
self.data_provider.from_db = True
self.data_provider.period = PRD.DAILY
self.config.pattern_type_list = FT.get_all()
self.config.detection_process = PDP.UPDATE_PATTERN_DATA
self.config.plot_data = False
self.config.with_trading = False
self.config.save_pattern_data = True
self.config.save_trade_data = False
self.config.save_wave_data = False
self.config.plot_only_pattern_with_fibonacci_waves = False
self.config.plot_min_max = False
self.config.plot_volume = False
self.config.length_for_local_min_max = 2
self.config.length_for_local_min_max_fibonacci = 1
self.config.bound_upper_value = CN.CLOSE
self.config.bound_lower_value = CN.CLOSE
def get_constraints_as_dict(sys_config: SystemConfiguration):
return_dict = {}
for pattern_type in FT.get_all():
return_dict[pattern_type] = ConstraintsFactory.get_constraints_by_pattern_type(pattern_type, sys_config)
return return_dict
def __get_length_pattern_type__(self) -> int:
pattern_to_check = '{}_{}'.format(
self._components_underscore[self._idx_pattern],
self._components_underscore[self._idx_pattern + 1])
return 2 if pattern_to_check in FT.get_all() else 1
sys_config.exchange_config.deactivate_automatic_trading()
sys_config.exchange_config.trade_strategy_dict = {
BT.BREAKOUT: [
TSTR.LIMIT, TSTR.LIMIT_FIX, TSTR.TRAILING_STEPPED_STOP,
TSTR.TRAILING_STOP
],
# BT.FC_TICKS: [TSTR.TRAILING_STOP]
}
sys_config.exchange_config.delete_vanished_patterns_from_trade_dict = False
sys_config.exchange_config.massive_breakout_pct = 5
sys_config.config.simple_moving_average_number = 20
sys_config.config.trading_last_price_mean_aggregation = 4
# debugger.pattern_range_position_list = [217, 224, 242]
sys_config.config.pattern_type_list = FT.get_all()
sys_config.config.with_trading = True
sys_config.config.save_pattern_data = True
sys_config.config.save_trade_data = True
sys_config.config.save_wave_data = False # they are saved within other threads
# sys_config.config.pattern_type_list = [FT.TRIANGLE]
sys_config.config.pattern_type_list = FT.get_all()
sys_config.config.plot_data = True
sys_config.config.plot_only_pattern_with_fibonacci_waves = False
sys_config.config.plot_min_max = True
sys_config.config.plot_volume = False
sys_config.config.length_for_local_min_max = 2
sys_config.config.length_for_local_min_max_fibonacci = 1
sys_config.config.statistics_excel_file_name = 'pattern_statistics/statistics_pattern_06-11.xlsx'
sys_config.config.statistics_excel_file_name = ''
sys_config.config.statistics_constraints_excel_file_name = ''
def get_pattern_records_for_replay_as_dataframe(self) -> pd.DataFrame:
query = self._pattern_table.get_query_select_for_records(
"Period = 'DAILY'")
db_df = DatabaseDataFrame(self, query)
return db_df.df[db_df.df[DC.PATTERN_TYPE].isin(
FT.get_long_trade_able_types())]
def __perform_task__(self):
self._db_updater.update_trade_policy_metric_for_today(
FT.get_long_trade_able_types())
"""
Description: This module contains the global system configuration classes for stock pattern detection
Author: Josef Sertl
Copyright: SERTL Analytics, https://sertl-analytics.com
Date: 2018-05-14
"""
from sertl_analytics.constants.pattern_constants import FT, BT, TSTR, TTC, TP, INDICES
from sertl_analytics.datafetcher.web_data_fetcher import IndicesComponentFetcher
from pattern_test.trade_test import TradeTest, TradeTestApi
api = TradeTestApi()
api.test_process = TP.BACK_TESTING
back_testing = TradeTest(api)
back_testing.sys_config.config.pattern_type_list = FT.get_long_trade_able_types(
)
# back_testing.sys_config.config.pattern_type_list = [FT.TRIANGLE_DOWN]
back_testing.sys_config.config.save_trade_data = True
back_testing.sys_config.config.simple_moving_average_number = 20
back_testing.sys_config.config.trading_last_price_mean_aggregation = 8
# ******** START setup **********
api.buy_trigger = BT.BREAKOUT
api.trade_strategy = TSTR.TRAILING_STOP
# ******** END setup **********
ticker_dic = IndicesComponentFetcher.get_ticker_name_dic(INDICES.DOW_JONES)
# ticker_dic = IndicesComponentFetcher.get_ticker_name_dic(INDICES.CRYPTO_CCY)
# ticker_dic = {'CVX': 'a'}
api.and_clause = "Date BETWEEN '2018-06-07' AND '2019-09-04'"
def _get_covered_pattern_types_() -> list:
return FT.get_head_shoulder_bottom_types()
def pattern_type(self):
return FT.get_pattern_type(int(self._id_components[4]))
def _get_covered_pattern_types_() -> list:
return FT.get_normal_types()
Pattern:
feature_columns=Ticks_Till_Pattern_Formed,Slope_Upper_PCT,Slope_Lower_PCT,Slope_Regression_PCT,Slope_Volume_Regression_PCT,Previous_Period_Half_Top_Out_PCT,Previous_Period_Full_Top_Out_PCT,Previous_Period_Half_Bottom_Out_PCT,Previous_Period_Full_Bottom_Out_PCT,Available_Fibonacci_Type_ID
label_columns=Touch_Points_Till_Breakout_Top,Touch_Points_Till_Breakout_Bottom
26,-2.0,-1.0,-1.0,-190.0,0.0,0.0,40.0,70.0,0
20,-1.0,-1.0,-1.0,10.0,50.0,50.0,0.0,20.0,0
"""
from pattern_predictor_optimizer import PatternPredictorOptimizer
from pattern_system_configuration import SystemConfiguration
from sertl_analytics.constants.pattern_constants import FT, PRED, STBL, MT, DC
sys_config = SystemConfiguration()
master_predictor_list = [
sys_config.master_predictor_for_trades,
sys_config.master_predictor_touch_points,
sys_config.master_predictor_before_breakout,
sys_config.master_predictor_after_breakout
]
for master_predictor in master_predictor_list:
patter_type_list = FT.get_all()
for pattern_type in patter_type_list:
print('Processing: {} - {}'.format(master_predictor.__class__.__name__,
pattern_type))
feature_columns = master_predictor.get_feature_columns(pattern_type)
x_data = [1 for col in feature_columns]
prediction_dict = master_predictor.predict_for_label_columns(
pattern_type, x_data)
print('{}: {}'.format(pattern_type, prediction_dict))