def _validate_stop_loss(self) -> None:
if self.stop_loss is None:
raise exceptions.InvalidStrategy(
'You forgot to set self.stop_loss. example [qty, price]')
elif type(self.stop_loss) not in [tuple, list, np.ndarray]:
raise exceptions.InvalidStrategy(
'self.stop_loss must be either a list or a tuple. example: [qty, price]'
)
def _validate_take_profit(self) -> None:
if self.take_profit is None:
raise exceptions.InvalidStrategy(
'You forgot to set self.take_profit. example [qty, price]')
elif type(self.take_profit) not in [tuple, list, np.ndarray]:
raise exceptions.InvalidStrategy(
'self.take_profit must be either a list or a tuple. example: [qty, price]'
)
def _on_open_position(self, order: Order) -> None:
self.increased_count = 1
self._broadcast('route-open-position')
if self.take_profit is not None:
for o in self._take_profit:
# validation: make sure take-profit will exit with profit
if self.is_long:
if o[1] <= self.position.entry_price:
raise exceptions.InvalidStrategy(
f'take-profit({o[1]}) must be above entry-price({self.position.entry_price}) in a long position'
)
elif self.is_short:
if o[1] >= self.position.entry_price:
raise exceptions.InvalidStrategy(
f'take-profit({o[1]}) must be below entry-price({self.position.entry_price}) in a short position'
)
# submit take-profit
submitted_order: Order = self.broker.reduce_position_at(
o[0],
o[1],
order_roles.CLOSE_POSITION
)
if submitted_order:
submitted_order.submitted_via = 'take-profit'
self._close_position_orders.append(submitted_order)
if self.stop_loss is not None:
for o in self._stop_loss:
# validation
if self.is_long:
if o[1] >= self.position.entry_price:
raise exceptions.InvalidStrategy(
f'stop-loss({o[1]}) must be below entry-price({self.position.entry_price}) in a long position'
)
elif self.is_short:
if o[1] <= self.position.entry_price:
raise exceptions.InvalidStrategy(
f'stop-loss({o[1]}) must be above entry-price({self.position.entry_price}) in a short position'
)
# submit stop-loss
submitted_order: Order = self.broker.reduce_position_at(
o[0],
o[1],
order_roles.CLOSE_POSITION
)
if submitted_order:
submitted_order.submitted_via = 'stop-loss'
self._close_position_orders.append(submitted_order)
self._open_position_orders = []
self.on_open_position(order)
self._detect_and_handle_entry_and_exit_modifications()
def _execute_long(self) -> None:
self.go_long()
# validation
if self.buy is None:
raise exceptions.InvalidStrategy(
'You forgot to set self.buy. example [qty, price]')
elif type(self.buy) not in [tuple, list]:
raise exceptions.InvalidStrategy(
'self.buy must be either a list or a tuple. example: [qty, price]'
)
self._prepare_buy()
if self.take_profit is not None:
# validate
self._validate_take_profit()
self._prepare_take_profit()
if self.stop_loss is not None:
# validate
self._validate_stop_loss()
self._prepare_stop_loss()
# filters
passed = self._execute_filters()
if not passed:
return
for o in self._buy:
# MARKET order
if abs(o[1] - self.price) < 0.0001:
submitted_order = self.broker.buy_at_market(
o[0], order_roles.OPEN_POSITION)
# STOP order
elif o[1] > self.price:
submitted_order = self.broker.start_profit_at(
sides.BUY, o[0], o[1], order_roles.OPEN_POSITION)
# LIMIT order
elif o[1] < self.price:
submitted_order = self.broker.buy_at(o[0], o[1],
order_roles.OPEN_POSITION)
else:
raise ValueError(
f'Invalid order price: o[1]:{o[1]}, self.price:{self.price}'
)
if submitted_order:
self._entry_orders.append(submitted_order)
def _execute_long(self):
self.go_long()
# validation
if self.buy is None:
raise exceptions.InvalidStrategy(
'You forgot to set self.buy. example [qty, price]')
elif type(self.buy) not in [tuple, list]:
raise exceptions.InvalidStrategy(
'self.buy must be either a list or a tuple. example: [qty, price]'
)
self._prepare_buy()
if self.take_profit is not None:
# validate
self._validate_take_profit()
self._prepare_take_profit()
if self.stop_loss is not None:
# validate
self._validate_stop_loss()
self._prepare_stop_loss()
# filters
for f in self.filters():
passed = f()
if passed is False:
logger.info(f.__name__)
self._reset()
return
for o in self._buy:
# STOP order
if o[1] > self.price:
self._open_position_orders.append(
self.broker.start_profit_at(sides.BUY, o[0], o[1],
order_roles.OPEN_POSITION))
# LIMIT order
elif o[1] < self.price:
self._open_position_orders.append(
self.broker.buy_at(o[0], o[1], order_roles.OPEN_POSITION))
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.buy_at_market(o[0], order_roles.OPEN_POSITION))
def _execute_filters(self):
for f in self.filters():
try:
passed = f()
except TypeError:
raise exceptions.InvalidStrategy(
"Invalid filter format. You need to pass filter methods WITHOUT calling them "
"(no parentheses must be present at the end)"
"\n\n"
u"\u274C " + "Incorrect Example:\n"
"return [\n"
" self.filter_1()\n"
"]\n\n"
u"\u2705 " + "Correct Example:\n"
"return [\n"
" self.filter_1\n"
"]\n"
)
if passed == False:
logger.info(f.__name__)
self._reset()
return False
return True
def average_take_profit(self) -> float:
if self._take_profit is None:
raise exceptions.InvalidStrategy(
'You cannot access self.average_take_profit before setting self.take_profit')
arr = self._take_profit
return (np.abs(arr[:, 0] * arr[:, 1])).sum() / np.abs(arr[:, 0]).sum()
def __init__(self, training_candles: ndarray, testing_candles: ndarray, optimal_total: int, cpu_cores: int, csv: bool,
json: bool, start_date: str, finish_date: str) -> None:
if len(router.routes) != 1:
raise NotImplementedError('optimize_mode mode only supports one route at the moment')
self.strategy_name = router.routes[0].strategy_name
self.optimal_total = optimal_total
self.exchange = router.routes[0].exchange
self.symbol = router.routes[0].symbol
self.timeframe = router.routes[0].timeframe
StrategyClass = jh.get_strategy_class(self.strategy_name)
self.strategy_hp = StrategyClass.hyperparameters(None)
solution_len = len(self.strategy_hp)
if solution_len == 0:
raise exceptions.InvalidStrategy('Targeted strategy does not implement a valid hyperparameters() method.')
super().__init__(
iterations=2000 * solution_len,
population_size=solution_len * 100,
solution_len=solution_len,
options={
'strategy_name': self.strategy_name,
'exchange': self.exchange,
'symbol': self.symbol,
'timeframe': self.timeframe,
'strategy_hp': self.strategy_hp,
'csv': csv,
'json': json,
'start_date': start_date,
'finish_date': finish_date,
}
)
if cpu_cores > cpu_count():
raise ValueError(f'Entered cpu cores number is more than available on this machine which is {cpu_count()}')
elif cpu_cores == 0:
self.cpu_cores = cpu_count()
else:
self.cpu_cores = cpu_cores
self.training_candles = training_candles
self.testing_candles = testing_candles
key = jh.key(self.exchange, self.symbol)
training_candles_start_date = jh.timestamp_to_time(self.training_candles[key]['candles'][0][0]).split('T')[0]
training_candles_finish_date = jh.timestamp_to_time(self.training_candles[key]['candles'][-1][0]).split('T')[0]
testing_candles_start_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][0][0]).split('T')[0]
testing_candles_finish_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][-1][0]).split('T')[0]
self.training_initial_candles = []
self.testing_initial_candles = []
for c in config['app']['considering_candles']:
self.training_initial_candles.append(
required_candles.load_required_candles(c[0], c[1], training_candles_start_date,
training_candles_finish_date))
self.testing_initial_candles.append(
required_candles.load_required_candles(c[0], c[1], testing_candles_start_date,
testing_candles_finish_date))
def _on_open_position(self):
logger.info('Detected open position')
self._broadcast('route-open-position')
if self.take_profit is not None:
for o in self._take_profit:
# validation: make sure take-profit will exit with profit
if self.is_long:
if o[1] <= self.position.entry_price:
raise exceptions.InvalidStrategy(
'take-profit({}) must be above entry-price({}) in a long position'
.format(o[1], self.position.entry_price))
elif self.is_short:
if o[1] >= self.position.entry_price:
raise exceptions.InvalidStrategy(
'take-profit({}) must be below entry-price({}) in a short position'
.format(o[1], self.position.entry_price))
# submit take-profit
self._take_profit_orders.append(
self.broker.reduce_position_at(o[0], o[1],
order_roles.CLOSE_POSITION))
if self.stop_loss is not None:
for o in self._stop_loss:
# validation
if self.is_long:
if o[1] >= self.position.entry_price:
raise exceptions.InvalidStrategy(
'stop-loss({}) must be below entry-price({}) in a long position'
.format(o[1], self.position.entry_price))
elif self.is_short:
if o[1] <= self.position.entry_price:
raise exceptions.InvalidStrategy(
'stop-loss({}) must be above entry-price({}) in a short position'
.format(o[1], self.position.entry_price))
# submit stop-loss
self._stop_loss_orders.append(
self.broker.stop_loss_at(o[0], o[1],
order_roles.CLOSE_POSITION))
self._open_position_orders = []
self._initial_qty = self.position.qty
self.on_open_position()
self._detect_and_handle_entry_and_exit_modifications()
def average_stop_loss(self) -> float:
if self._stop_loss is None:
raise exceptions.InvalidStrategy(
'You cannot access self.average_stop_loss before setting self.stop_loss'
)
arr = self._stop_loss
return (np.abs(arr[:, 0] * arr[:, 1])).sum() / np.abs(arr[:, 0]).sum()
def _execute_short(self) -> None:
self.go_short()
# validation
if self.sell is None:
raise exceptions.InvalidStrategy(
'You forgot to set self.sell. example [qty, price]')
elif type(self.sell) not in [tuple, list]:
raise exceptions.InvalidStrategy(
'self.sell must be either a list or a tuple. example: [qty, price]'
)
self._prepare_sell()
if self.take_profit is not None:
self._validate_take_profit()
self._prepare_take_profit()
if self.stop_loss is not None:
self._validate_stop_loss()
self._prepare_stop_loss()
# filters
passed = self._execute_filters()
if not passed:
return
for o in self._sell:
# STOP order
if o[1] < self.price:
self._open_position_orders.append(
self.broker.start_profit_at(sides.SELL, o[0], o[1],
order_roles.OPEN_POSITION))
# LIMIT order
elif o[1] > self.price:
self._open_position_orders.append(
self.broker.sell_at(o[0], o[1], order_roles.OPEN_POSITION))
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.sell_at_market(o[0],
order_roles.OPEN_POSITION))
def __init__(self, training_candles, testing_candles, optimal_total):
if len(router.routes) != 1:
raise NotImplementedError(
'optimize_mode mode only supports one route at the moment')
self.strategy_name = router.routes[0].strategy_name
self.optimal_total = optimal_total
self.exchange = router.routes[0].exchange
self.symbol = router.routes[0].symbol
self.timeframe = router.routes[0].timeframe
StrategyClass = jh.get_strategy_class(self.strategy_name)
self.strategy_hp = StrategyClass.hyperparameters(None)
solution_len = len(self.strategy_hp)
if solution_len == 0:
raise exceptions.InvalidStrategy(
'Targeted strategy does not implement a valid hyperparameters() method.'
)
super().__init__(iterations=2000 * solution_len,
population_size=solution_len * 100,
solution_len=solution_len,
options={
'strategy_name': self.strategy_name,
'exchange': self.exchange,
'symbol': self.symbol,
'timeframe': self.timeframe
})
self.training_candles = training_candles
self.testing_candles = testing_candles
key = jh.key(self.exchange, self.symbol)
training_candles_start_date = jh.timestamp_to_time(
self.training_candles[key]['candles'][0][0]).split('T')[0]
training_candles_finish_date = jh.timestamp_to_time(
self.training_candles[key]['candles'][-1][0]).split('T')[0]
testing_candles_start_date = jh.timestamp_to_time(
self.testing_candles[key]['candles'][0][0]).split('T')[0]
testing_candles_finish_date = jh.timestamp_to_time(
self.testing_candles[key]['candles'][-1][0]).split('T')[0]
self.training_initial_candles = []
self.testing_initial_candles = []
for c in config['app']['considering_candles']:
self.training_initial_candles.append(
required_candles.load_required_candles(
c[0], c[1], training_candles_start_date,
training_candles_finish_date))
self.testing_initial_candles.append(
required_candles.load_required_candles(
c[0], c[1], testing_candles_start_date,
testing_candles_finish_date))
def simulator(candles: Dict[str, Dict[str, Union[str, np.ndarray]]],
hyperparameters=None) -> None:
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# inject hyper parameters (used for optimize_mode)
# convert DNS string into hyperparameters
if r.dna and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.dna)
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(
previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def simulator(
candles: dict, run_silently: bool, hyperparameters: dict = None
) -> None:
begin_time_track = time.time()
key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}"
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
try:
store.app.starting_time = first_candles_set[0][0]
except IndexError:
raise IndexError('Check your "warm_up_candles" config value')
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
# if the r.strategy is str read it from file
if isinstance(r.strategy_name, str):
StrategyClass = jh.get_strategy_class(r.strategy_name)
# else it is a class object so just use it
else:
StrategyClass = r.strategy_name
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# read the dna from strategy's dna() and use it for injecting inject hyperparameters
# first convert DNS string into hyperparameters
if len(r.strategy.dna()) > 0 and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.strategy.dna())
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
progressbar = Progressbar(length, step=60)
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle, exchange, symbol, '1m', with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
# until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle, exchange, symbol, timeframe, with_execution=False,
with_generation=False)
# update progressbar
if not run_silently and i % 60 == 0:
progressbar.update()
sync_publish('progressbar', {
'current': progressbar.current,
'estimated_remaining_seconds': progressbar.estimated_remaining_seconds
})
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(store.candles.get_current_candle(r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not run_silently:
# print executed time for the backtest session
finish_time_track = time.time()
sync_publish('alert', {
'message': f'Successfully executed backtest simulation in: {round(finish_time_track - begin_time_track, 2)} seconds',
'type': 'success'
})
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def __init__(
self,
training_candles: ndarray, testing_candles: ndarray,
optimal_total: int, cpu_cores: int,
csv: bool,
export_json: bool,
start_date: str, finish_date: str,
charset: str = r'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw',
fitness_goal: float = 1,
) -> None:
if len(router.routes) != 1:
raise NotImplementedError('optimize_mode mode only supports one route at the moment')
self.strategy_name = router.routes[0].strategy_name
self.exchange = router.routes[0].exchange
self.symbol = router.routes[0].symbol
self.timeframe = router.routes[0].timeframe
strategy_class = jh.get_strategy_class(self.strategy_name)
self.strategy_hp = strategy_class.hyperparameters(None)
solution_len = len(self.strategy_hp)
if solution_len == 0:
raise exceptions.InvalidStrategy('Targeted strategy does not implement a valid hyperparameters() method.')
self.started_index = 0
self.start_time = jh.now_to_timestamp()
self.population = []
self.iterations = 2000 * solution_len
self.population_size = solution_len * 100
self.solution_len = solution_len
self.charset = charset
self.fitness_goal = fitness_goal
self.cpu_cores = 0
self.optimal_total = optimal_total
self.training_candles = training_candles
self.testing_candles = testing_candles
self.average_execution_seconds = 0
# check for termination event once per second
tl_0 = Timeloop()
@tl_0.job(interval=timedelta(seconds=1))
def check_for_termination():
if process_status() != 'started':
raise exceptions.Termination
tl_0.start()
options = {
'strategy_name': self.strategy_name,
'exchange': self.exchange,
'symbol': self.symbol,
'timeframe': self.timeframe,
'strategy_hp': self.strategy_hp,
'csv': csv,
'json': export_json,
'start_date': start_date,
'finish_date': finish_date,
}
self.options = {} if options is None else options
os.makedirs('./storage/temp/optimize', exist_ok=True)
self.temp_path = f"./storage/temp/optimize/{self.options['strategy_name']}-{self.options['exchange']}-{self.options['symbol']}-{self.options['timeframe']}-{self.options['start_date']}-{self.options['finish_date']}.pickle"
if fitness_goal > 1 or fitness_goal < 0:
raise ValueError('fitness scores must be between 0 and 1')
if not optimal_total > 0:
raise ValueError('optimal_total must be bigger than 0')
# # if temp file exists, load data to resume previous session
# if jh.file_exists(self.temp_path) and click.confirm(
# 'Previous session detected. Do you want to resume?', default=True
# ):
# self.load_progress()
if cpu_cores > cpu_count():
raise ValueError(f'Entered cpu cores number is more than available on this machine which is {cpu_count()}')
elif cpu_cores == 0:
self.cpu_cores = cpu_count()
else:
self.cpu_cores = cpu_cores
def evolve(self) -> list:
"""
the main method, that runs the evolutionary algorithm
"""
# clear the logs to start from a clean slate
jh.clear_file('storage/logs/optimize-mode.txt')
logger.log_optimize_mode('Optimization session started')
if self.started_index == 0:
logger.log_optimize_mode(
f"Generating {self.population_size} population size (random DNAs) using {self.cpu_cores} CPU cores"
)
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
msg = f'Too many errors! less than half of the expected population size could be generated. Only {len(self.population)} indviduals from planned {self.population_size} are usable.'
logger.log_optimize_mode(msg)
raise ValueError(msg)
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
msg = 'Cannot continue because no individual with the minimum fitness-score was found. Your strategy seems to be flawed or maybe it requires modifications. '
logger.log_optimize_mode(msg)
raise exceptions.InvalidStrategy(msg)
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
progressbar = Progressbar(loop_length)
while i < loop_length:
with Manager() as manager:
people_bucket = manager.list([])
workers = []
try:
for _ in range(self.cpu_cores):
mommy = self.select_person()
daddy = self.select_person()
w = Process(
target=create_baby,
args=(
people_bucket, mommy, daddy, self.solution_len, self.charset,
jh.get_config('env.optimization'), router.formatted_routes,
router.formatted_extra_routes,
self.strategy_hp, self.training_candles, self.testing_candles,
self.optimal_total
)
)
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {w.exitcode}')
except exceptions.Termination:
self._handle_termination(manager, workers)
# update dashboard
click.clear()
self.update_progressbar(progressbar)
# general_info streams
general_info = {
'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
'index': f'{(i + 1) * self.cpu_cores}/{self.iterations}',
'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
'average_execution_seconds': self.average_execution_seconds
}
if jh.is_debugging():
general_info['population_size'] = self.population_size
general_info['iterations'] = self.iterations
general_info['solution_length'] = self.solution_len
sync_publish('general_info', general_info)
if self.population_size > 50:
number_of_ind_to_show = 40
elif self.population_size > 20:
number_of_ind_to_show = 15
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
best_candidates = [{
'rank': j + 1,
'dna': self.population[j]['dna'],
'fitness': round(self.population[j]['fitness'], 4),
'training_win_rate': self.population[j]['training_log']['win-rate'],
'training_total_trades': self.population[j]['training_log']['total'],
'training_pnl': self.population[j]['training_log']['PNL'],
'testing_win_rate': self.population[j]['testing_log']['win-rate'],
'testing_total_trades': self.population[j]['testing_log']['total'],
'testing_pnl': self.population[j]['testing_log']['PNL'],
} for j in range(number_of_ind_to_show)]
sync_publish('best_candidates', best_candidates)
# one person has to die and be replaced with the newborn baby
for baby in people_bucket:
# never kill our best performer
random_index = randint(1, len(self.population) - 1)
self.population[random_index] = baby
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
self.update_progressbar(progressbar, finished=True)
sync_publish('alert', {
'message': f'Fitness goal reached after iteration {i*self.cpu_cores}',
'type': 'success'
})
return baby
# TODO: bring back progress resumption
# # save progress after every n iterations
# if i != 0 and int(i * self.cpu_cores) % 50 == 0:
# self.save_progress(i)
# TODO: bring back
# # store a take_snapshot of the fittest individuals of the population
# if i != 0 and i % int(100 / self.cpu_cores) == 0:
# self.take_snapshot(i * self.cpu_cores)
i += 1
sync_publish('alert', {
'message': f"Finished {self.iterations} iterations. Check your best DNA candidates, "
f"if you don't like any of them, try modifying your strategy.",
'type': 'success'
})
return self.population
def _detect_and_handle_entry_and_exit_modifications(self) -> None:
if self.position.is_close:
return
try:
if self.is_long:
# prepare format
self._prepare_buy(make_copies=False)
# if entry has been modified
if not np.array_equal(self.buy, self._buy):
self._buy = self.buy.copy()
# cancel orders
for o in self._open_position_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._open_position_orders = [
o for o in self._open_position_orders if o.is_executed
]
for o in self._buy:
# STOP order
if o[1] > self.price:
self._open_position_orders.append(
self.broker.start_profit_at(
sides.BUY, o[0], o[1],
order_roles.OPEN_POSITION))
# LIMIT order
elif o[1] < self.price:
self._open_position_orders.append(
self.broker.buy_at(o[0], o[1],
order_roles.OPEN_POSITION))
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.buy_at_market(
o[0], order_roles.OPEN_POSITION))
elif self.is_short:
# prepare format
self._prepare_sell(make_copies=False)
# if entry has been modified
if not np.array_equal(self.sell, self._sell):
self._sell = self.sell.copy()
# cancel orders
for o in self._open_position_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._open_position_orders = [
o for o in self._open_position_orders if o.is_executed
]
for o in self._sell:
# STOP order
if o[1] < self.price:
self._open_position_orders.append(
self.broker.start_profit_at(
sides.SELL, o[0], o[1],
order_roles.OPEN_POSITION))
# LIMIT order
elif o[1] > self.price:
self._open_position_orders.append(
self.broker.sell_at(o[0], o[1],
order_roles.OPEN_POSITION))
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.sell_at_market(
o[0], order_roles.OPEN_POSITION))
if self.position.is_open and self.take_profit is not None:
self._validate_take_profit()
self._prepare_take_profit(False)
# if _take_profit has been modified
if not np.array_equal(self.take_profit, self._take_profit):
self._take_profit = self.take_profit.copy()
# cancel orders
for o in self._take_profit_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._take_profit_orders = [
o for o in self._take_profit_orders if o.is_executed
]
self._log_take_profit = []
for s in self._take_profit_orders:
self._log_take_profit.append((abs(s.qty), s.price))
for o in self._take_profit:
self._log_take_profit.append(o)
self._take_profit_orders.append(
self.broker.reduce_position_at(
o[0], o[1], order_roles.CLOSE_POSITION))
if self.position.is_open and self.stop_loss is not None:
self._validate_stop_loss()
self._prepare_stop_loss(False)
# if stop_loss has been modified
if not np.array_equal(self.stop_loss, self._stop_loss):
# prepare format
self._stop_loss = self.stop_loss.copy()
# cancel orders
for o in self._stop_loss_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._stop_loss_orders = [
o for o in self._stop_loss_orders if o.is_executed
]
self._log_stop_loss = []
for s in self._stop_loss_orders:
self._log_stop_loss.append((abs(s.qty), s.price))
for o in self._stop_loss:
self._log_stop_loss.append(o)
self._stop_loss_orders.append(
self.broker.reduce_position_at(
o[0], o[1], order_roles.CLOSE_POSITION))
except TypeError:
raise exceptions.InvalidStrategy(
'Something odd is going on with your strategy. '
'Try running it with "--debug" to see what was going on near the end, and fix it.'
)
except:
raise
# validations: stop-loss and take-profit should not be the same
if self.position.is_open:
if (self.stop_loss is not None
and self.take_profit is not None) and np.array_equal(
self.stop_loss, self.take_profit):
raise exceptions.InvalidStrategy(
'stop-loss and take-profit should not be exactly the same. Just use either one of them and it will do.'
)
def _detect_and_handle_entry_and_exit_modifications(self) -> None:
if self.position.is_close:
return
try:
if self.is_long:
# prepare format
self._prepare_buy(make_copies=False)
# if entry has been modified
if not np.array_equal(self.buy, self._buy):
self._buy = self.buy.copy()
# cancel orders
for o in self._entry_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
self._entry_orders = [
o for o in self._entry_orders if o.is_executed
]
for o in self._buy:
# MARKET order
if abs(o[1] - self.price) < 0.0001:
submitted_order = self.broker.buy_at_market(
o[0], order_roles.OPEN_POSITION)
# STOP order
elif o[1] > self.price:
submitted_order = self.broker.start_profit_at(
sides.BUY, o[0], o[1],
order_roles.OPEN_POSITION)
# LIMIT order
elif o[1] < self.price:
submitted_order = self.broker.buy_at(
o[0], o[1], order_roles.OPEN_POSITION)
else:
raise ValueError(
f'Invalid order price: o[1]:{o[1]}, self.price:{self.price}'
)
if submitted_order:
self._entry_orders.append(submitted_order)
elif self.is_short:
# prepare format
self._prepare_sell(make_copies=False)
# if entry has been modified
if not np.array_equal(self.sell, self._sell):
self._sell = self.sell.copy()
# cancel orders
for o in self._entry_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
self._entry_orders = [
o for o in self._entry_orders if o.is_executed
]
for o in self._sell:
# MARKET order
if abs(o[1] - self.price) < 0.0001:
submitted_order = self.broker.sell_at_market(
o[0], order_roles.OPEN_POSITION)
# STOP order
elif o[1] < self.price:
submitted_order = self.broker.start_profit_at(
sides.SELL, o[0], o[1],
order_roles.OPEN_POSITION)
# LIMIT order
elif o[1] > self.price:
submitted_order = self.broker.sell_at(
o[0], o[1], order_roles.OPEN_POSITION)
else:
raise ValueError(
f'Invalid order price: o[1]:{o[1]}, self.price:{self.price}'
)
if submitted_order:
self._entry_orders.append(submitted_order)
if self.position.is_open and self.take_profit is not None:
self._validate_take_profit()
self._prepare_take_profit(False)
# if _take_profit has been modified
if not np.array_equal(self.take_profit, self._take_profit):
self._take_profit = self.take_profit.copy()
# cancel orders
for o in self._exit_orders:
if o.submitted_via == 'take-profit' and o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# remove canceled orders to optimize the loop
self._exit_orders = [
o for o in self._exit_orders if not o.is_canceled
]
for o in self._take_profit:
submitted_order: Order = self.broker.reduce_position_at(
o[0], o[1], order_roles.CLOSE_POSITION)
if submitted_order:
submitted_order.submitted_via = 'take-profit'
self._exit_orders.append(submitted_order)
if self.position.is_open and self.stop_loss is not None:
self._validate_stop_loss()
self._prepare_stop_loss(False)
# if stop_loss has been modified
if not np.array_equal(self.stop_loss, self._stop_loss):
# prepare format
self._stop_loss = self.stop_loss.copy()
# cancel orders
for o in self._exit_orders:
if o.submitted_via == 'stop-loss' and o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# remove canceled orders to optimize the loop
self._exit_orders = [
o for o in self._exit_orders if not o.is_canceled
]
for o in self._stop_loss:
submitted_order: Order = self.broker.reduce_position_at(
o[0], o[1], order_roles.CLOSE_POSITION)
if submitted_order:
submitted_order.submitted_via = 'stop-loss'
self._exit_orders.append(submitted_order)
except TypeError:
raise exceptions.InvalidStrategy(
'Something odd is going on within your strategy causing a TypeError exception. '
'Try running it with "--debug" in a backtest to see what was going on near the end, and fix it.'
)
except:
raise
# validations: stop-loss and take-profit should not be the same
if (self.position.is_open and
(self.stop_loss is not None and self.take_profit is not None)
and np.array_equal(self.stop_loss, self.take_profit)):
raise exceptions.InvalidStrategy(
'stop-loss and take-profit should not be exactly the same. Just use either one of them and it will do.'
)
def _detect_and_handle_entry_and_exit_modifications(self):
if self.position.is_close:
return
if self.is_long:
# prepare format
if type(self.buy[0]) not in [list, tuple, np.ndarray]:
self.buy = [self.buy]
self.buy = np.array(self.buy, dtype=float)
# if entry has been modified
if not np.array_equal(self.buy, self._buy):
self._buy = self.buy.copy()
# cancel orders
for o in self._open_position_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._open_position_orders = [o for o in self._open_position_orders if o.is_executed]
for o in self._buy:
# STOP order
if o[1] > self.price:
self._open_position_orders.append(
self.broker.start_profit_at(sides.BUY, o[0], o[1], order_roles.OPEN_POSITION)
)
# LIMIT order
elif o[1] < self.price:
self._open_position_orders.append(
self.broker.buy_at(o[0], o[1], order_roles.OPEN_POSITION)
)
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.buy_at_market(o[0], order_roles.OPEN_POSITION)
)
elif self.is_short:
# prepare format
if type(self.sell[0]) not in [list, tuple, np.ndarray]:
self.sell = [self.sell]
self.sell = np.array(self.sell, dtype=float)
# if entry has been modified
if not np.array_equal(self.sell, self._sell):
self._sell = self.sell.copy()
# cancel orders
for o in self._open_position_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._open_position_orders = [o for o in self._open_position_orders if o.is_executed]
for o in self._sell:
# STOP order
if o[1] > self.price:
self._open_position_orders.append(
self.broker.start_profit_at(sides.BUY, o[0], o[1], order_roles.OPEN_POSITION)
)
# LIMIT order
elif o[1] < self.price:
self._open_position_orders.append(
self.broker.sell_at(o[0], o[1], order_roles.OPEN_POSITION)
)
# MARKET order
elif o[1] == self.price:
self._open_position_orders.append(
self.broker.sell_at_market(o[0], order_roles.OPEN_POSITION)
)
if self.position.is_open and self.take_profit is not None:
self._validate_take_profit()
self._prepare_take_profit(False)
# if _take_profit has been modified
if not np.array_equal(self.take_profit, self._take_profit):
self._take_profit = self.take_profit.copy()
# cancel orders
for o in self._take_profit_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._take_profit_orders = [o for o in self._take_profit_orders if o.is_executed]
self._log_take_profit = []
for s in self._take_profit_orders:
self._log_take_profit.append(
(abs(s.qty), s.price)
)
for o in self._take_profit:
self._log_take_profit.append(o)
if o[1] == self.price:
if self.is_long:
self._take_profit_orders.append(
self.broker.sell_at_market(o[0], role=order_roles.CLOSE_POSITION)
)
elif self.is_short:
self._take_profit_orders.append(
self.broker.buy_at_market(o[0], role=order_roles.CLOSE_POSITION)
)
else:
if (self.is_long and o[1] > self.price) or (self.is_short and o[1] < self.price):
self._take_profit_orders.append(
self.broker.reduce_position_at(
o[0],
o[1],
order_roles.CLOSE_POSITION
)
)
elif (self.is_long and o[1] < self.price) or (self.is_short and o[1] > self.price):
self._take_profit_orders.append(
self.broker.stop_loss_at(
o[0],
o[1],
order_roles.CLOSE_POSITION
)
)
if self.position.is_open and self.stop_loss is not None:
self._validate_stop_loss()
self._prepare_stop_loss(False)
# if stop_loss has been modified
if not np.array_equal(self.stop_loss, self._stop_loss):
# prepare format
self._stop_loss = self.stop_loss.copy()
# cancel orders
for o in self._stop_loss_orders:
if o.is_active or o.is_queued:
self.broker.cancel_order(o.id)
# clean orders array but leave executed ones
self._stop_loss_orders = [o for o in self._stop_loss_orders if o.is_executed]
self._log_stop_loss = []
for s in self._stop_loss_orders:
self._log_stop_loss.append(
(abs(s.qty), s.price)
)
for o in self._stop_loss:
self._log_stop_loss.append(o)
if o[1] == self.price:
if self.is_long:
self._stop_loss_orders.append(
self.broker.sell_at_market(o[0], role=order_roles.CLOSE_POSITION)
)
elif self.is_short:
self._stop_loss_orders.append(
self.broker.buy_at_market(o[0], role=order_roles.CLOSE_POSITION)
)
else:
self._stop_loss_orders.append(
self.broker.stop_loss_at(
o[0],
o[1],
order_roles.CLOSE_POSITION
)
)
# validations: stop-loss and take-profit should not be the same
if self.position.is_open:
if (self.stop_loss is not None and self.take_profit is not None) and np.array_equal(self.stop_loss,
self.take_profit):
raise exceptions.InvalidStrategy(
'stop-loss and take-profit should not be exactly the same. Just use either one of them and it will do.')
