def _terminate(self) -> None:
"""
Optional for executing code after completion of a backTest.
This block will not execute in live use as a live
Jesse is never ending.
"""
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Terminating strategy...")
self.terminate()
self._detect_and_handle_entry_and_exit_modifications()
# fake execution of market orders in backtest simulation
if not jh.is_live():
store.orders.execute_pending_market_orders()
if jh.is_live():
return
if self.position.is_open:
store.app.total_open_trades += 1
store.app.total_open_pl += self.position.pnl
logger.info(
f"Closed open {self.exchange}-{self.symbol} position at {self.position.current_price} with PNL: {round(self.position.pnl, 4)}({round(self.position.pnl_percentage, 2)}%) because we reached the end of the backtest session."
)
# fake a closing (market) order so that the calculations would be correct
self.broker.reduce_position_at(self.position.qty, self.position.current_price, order_roles.CLOSE_POSITION)
return
if self._open_position_orders:
self._execute_cancel()
logger.info('Canceled open-position orders because we reached the end of the backtest session.')
def error(msg: str) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
# error logs should be logged as info logs as well
info(msg)
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
# notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify_urgently(f"ERROR:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data() or jh.is_live():
sync_publish('error_log', log_dict)
store.logs.errors.append(log_dict)
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.error(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'error')
def info(msg: str, send_notification=False) -> None:
if jh.app_mode() not in LOGGERS:
_init_main_logger()
msg = str(msg)
from jesse.store import store
log_id = jh.generate_unique_id()
log_dict = {
'id': log_id,
'timestamp': jh.now_to_timestamp(),
'message': msg
}
store.logs.info.append(log_dict)
if jh.is_collecting_data() or jh.is_live():
sync_publish('info_log', log_dict)
if jh.is_live() or (jh.is_backtesting() and jh.is_debugging()):
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logger = LOGGERS[jh.app_mode()]
logger.info(msg)
if jh.is_live():
from jesse.models.utils import store_log_into_db
store_log_into_db(log_dict, 'info')
if send_notification:
notify(msg)
def _on_take_profit(self, order: Order) -> None:
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Take-profit order has been executed.")
self._broadcast('route-take-profit')
self._execute_cancel()
self.on_take_profit(order)
self._detect_and_handle_entry_and_exit_modifications()
def _on_increased_position(self):
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Position size increased.")
self._broadcast('route-increased-position')
self.on_increased_position()
self._detect_and_handle_entry_and_exit_modifications()
def _on_close_position(self, order: Order):
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("A closing order has been executed")
self._broadcast('route-close-position')
self._execute_cancel()
self.on_close_position(order)
self._detect_and_handle_entry_and_exit_modifications()
def _on_stop_loss(self, order: Order) -> None:
if not jh.should_execute_silently() or jh.is_debugging():
logger.info('Stop-loss has been executed.')
self._broadcast('route-stop-loss')
self._execute_cancel()
self.on_stop_loss(order)
self._detect_and_handle_entry_and_exit_modifications()
def _check(self) -> None:
"""Based on the newly updated info, check if we should take action or not"""
if not self._is_initiated:
self._is_initiated = True
if jh.is_live() and jh.is_debugging():
logger.info(
f'Executing {self.name}-{self.exchange}-{self.symbol}-{self.timeframe}'
)
# for caution to make sure testing on livetrade won't bleed your account
if jh.is_test_driving() and store.completed_trades.count >= 2:
logger.info('Maximum allowed trades in test-drive mode is reached')
return
if self._open_position_orders != [] and self.is_close and self.should_cancel(
):
self._execute_cancel()
# make sure order cancellation response is received via WS
if jh.is_live():
# sleep a little until cancel is received via WS
sleep(0.1)
# just in case, sleep some more if necessary
for _ in range(20):
if store.orders.count_active_orders(
self.exchange, self.symbol) == 0:
break
logger.info('sleeping 0.2 more seconds...')
sleep(0.2)
# If it's still not cancelled, something is wrong. Handle cancellation failure
if store.orders.count_active_orders(self.exchange,
self.symbol) != 0:
raise exceptions.ExchangeNotResponding(
'The exchange did not respond as expected')
if self.position.is_open:
self._update_position()
if jh.is_backtesting() or jh.is_unit_testing():
store.orders.execute_pending_market_orders()
if self.position.is_close and self._open_position_orders == []:
should_short = self.should_short()
should_long = self.should_long()
# validation
if should_short and should_long:
raise exceptions.ConflictingRules(
'should_short and should_long should not be true at the same time.'
)
if should_long:
self._execute_long()
elif should_short:
self._execute_short()
def info(msg: str) -> None:
from jesse.store import store
store.logs.info.append({'time': jh.now_to_timestamp(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}')
if jh.is_live():
msg = f"[INFO | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {str(msg)}"
logging.info(msg)
def _on_reduced_position(self):
"""
prepares for on_reduced_position() is implemented by user
"""
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Position size reduced.")
self._broadcast('route-reduced-position')
self.on_reduced_position()
self._detect_and_handle_entry_and_exit_modifications()
def info(msg):
from jesse.store import store
store.logs.info.append({'time': jh.now(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta'))
if jh.is_live():
msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.info(msg)
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red'))
store.logs.errors.append({'time': jh.now(), 'message': msg})
if jh.is_live():
msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.error(msg)
def _on_take_profit(self, order: Order) -> None:
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Take-profit order has been executed.")
self._broadcast('route-take-profit')
self._execute_cancel()
# set metrics only after a trade happens and they actually change
self.metrics = metrics.trades(store.completed_trades.trades,
store.app.daily_balance)
self.on_take_profit(order)
self._detect_and_handle_entry_and_exit_modifications()
def error(msg: str) -> None:
msg = str(msg)
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}', 'red'))
store.logs.errors.append({'time': jh.now_to_timestamp(), 'message': msg})
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logging.error(msg)
def _terminate(self):
"""
Optional for executing code after completion of a backTest.
This block will not execute in live use as a live
Jesse is never ending.
"""
if not jh.should_execute_silently() or jh.is_debugging():
logger.info("Terminating strategy...")
self.terminate()
self._detect_and_handle_entry_and_exit_modifications()
# fake execution of market orders in backtest simulation
if not jh.is_live():
store.orders.execute_pending_market_orders()
if jh.is_live():
return
if self.position.is_open:
store.app.total_open_trades += 1
store.app.total_open_pl += self.position.pnl
logger.info(
"Closed open {}-{} position at {} with PNL: {}({}%) because we reached the end of the backtest session."
.format(self.exchange, self.symbol,
self.position.current_price, self.position.pnl,
self.position.pnl_percentage))
self.position._close(self.position.current_price)
self._execute_cancel()
return
if self._open_position_orders:
self._execute_cancel()
logger.info(
'Canceled open-position orders because we reached the end of the backtest session.'
)
def test_is_debugging():
assert jh.is_debugging() is False
def generate_initial_population(self) -> None:
"""
generates the initial population
"""
loop_length = int(self.population_size / self.cpu_cores)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna: str, dna_bucket: list) -> None:
try:
fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'training_log': d[2],
'testing_log': d[3]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index', f'{len(self.population)}/{self.population_size}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-' * 10, '-' * 10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]:
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError('Too many errors: less then half of the planned population size could be generated.')
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
jh.terminate_app()
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people: List) -> None:
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
]
if jh.is_debugging():
table_items.insert(
3,
['Population Size, Solution Length',
f'{self.population_size}, {self.solution_len}']
)
table.key_value(table_items, 'info', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
if jh.is_debugging():
fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
else:
fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
'PNL'] > 0 and self.population[j]['testing_log'][
'PNL'] > 0:
log = jh.style(log, 'bold')
if jh.is_debugging():
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
self.population[j]['fitness'],
log
],
)
else:
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
log
],
)
if jh.is_debugging():
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
else:
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(1, len(self.population) - 1) # never kill our best perforemr
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print(f'self.population_size: {self.population_size}')
print(f'self.population length: {len(self.population)}')
jh.terminate_app()
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print(f'fitness goal reached after iteration {i}')
return baby
# save progress after every n iterations
if i != 0 and int(i * self.cpu_cores) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / self.cpu_cores) == 0:
self.take_snapshot(i * self.cpu_cores)
i += 1
print('\n\n')
print(f'Finished {self.iterations} iterations.')
return self.population
def evolve(self) -> list:
"""
the main method, that runs the evolutionary algorithm
"""
# clear the logs to start from a clean slate
jh.clear_file('storage/logs/optimize-mode.txt')
logger.log_optimize_mode('Optimization session started')
if self.started_index == 0:
logger.log_optimize_mode(
f"Generating {self.population_size} population size (random DNAs) using {self.cpu_cores} CPU cores"
)
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
msg = f'Too many errors! less than half of the expected population size could be generated. Only {len(self.population)} indviduals from planned {self.population_size} are usable.'
logger.log_optimize_mode(msg)
raise ValueError(msg)
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
msg = 'Cannot continue because no individual with the minimum fitness-score was found. Your strategy seems to be flawed or maybe it requires modifications. '
logger.log_optimize_mode(msg)
raise exceptions.InvalidStrategy(msg)
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
progressbar = Progressbar(loop_length)
while i < loop_length:
with Manager() as manager:
people_bucket = manager.list([])
workers = []
try:
for _ in range(self.cpu_cores):
mommy = self.select_person()
daddy = self.select_person()
w = Process(
target=create_baby,
args=(
people_bucket, mommy, daddy, self.solution_len, self.charset,
jh.get_config('env.optimization'), router.formatted_routes,
router.formatted_extra_routes,
self.strategy_hp, self.training_candles, self.testing_candles,
self.optimal_total
)
)
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {w.exitcode}')
except exceptions.Termination:
self._handle_termination(manager, workers)
# update dashboard
click.clear()
self.update_progressbar(progressbar)
# general_info streams
general_info = {
'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
'index': f'{(i + 1) * self.cpu_cores}/{self.iterations}',
'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
'average_execution_seconds': self.average_execution_seconds
}
if jh.is_debugging():
general_info['population_size'] = self.population_size
general_info['iterations'] = self.iterations
general_info['solution_length'] = self.solution_len
sync_publish('general_info', general_info)
if self.population_size > 50:
number_of_ind_to_show = 40
elif self.population_size > 20:
number_of_ind_to_show = 15
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
best_candidates = [{
'rank': j + 1,
'dna': self.population[j]['dna'],
'fitness': round(self.population[j]['fitness'], 4),
'training_win_rate': self.population[j]['training_log']['win-rate'],
'training_total_trades': self.population[j]['training_log']['total'],
'training_pnl': self.population[j]['training_log']['PNL'],
'testing_win_rate': self.population[j]['testing_log']['win-rate'],
'testing_total_trades': self.population[j]['testing_log']['total'],
'testing_pnl': self.population[j]['testing_log']['PNL'],
} for j in range(number_of_ind_to_show)]
sync_publish('best_candidates', best_candidates)
# one person has to die and be replaced with the newborn baby
for baby in people_bucket:
# never kill our best performer
random_index = randint(1, len(self.population) - 1)
self.population[random_index] = baby
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
self.update_progressbar(progressbar, finished=True)
sync_publish('alert', {
'message': f'Fitness goal reached after iteration {i*self.cpu_cores}',
'type': 'success'
})
return baby
# TODO: bring back progress resumption
# # save progress after every n iterations
# if i != 0 and int(i * self.cpu_cores) % 50 == 0:
# self.save_progress(i)
# TODO: bring back
# # store a take_snapshot of the fittest individuals of the population
# if i != 0 and i % int(100 / self.cpu_cores) == 0:
# self.take_snapshot(i * self.cpu_cores)
i += 1
sync_publish('alert', {
'message': f"Finished {self.iterations} iterations. Check your best DNA candidates, "
f"if you don't like any of them, try modifying your strategy.",
'type': 'success'
})
return self.population
def generate_initial_population(self) -> None:
"""
generates the initial population
"""
length = int(self.population_size / self.cpu_cores)
progressbar = Progressbar(length)
for i in range(length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
try:
for _ in range(self.cpu_cores):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(
target=get_and_add_fitness_to_the_bucket,
args=(
dna_bucket, jh.get_config('env.optimization'), router.formatted_routes, router.formatted_extra_routes,
self.strategy_hp, dna, self.training_candles, self.testing_candles,
self.optimal_total
)
)
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {w.exitcode}')
except exceptions.Termination:
self._handle_termination(manager, workers)
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'training_log': d[2],
'testing_log': d[3]
})
# update dashboard
self.update_progressbar(progressbar)
# general_info streams
general_info = {
'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
'index': f'{(i + 1) * self.cpu_cores}/{self.population_size}',
'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
'average_execution_seconds': self.average_execution_seconds
}
if jh.is_debugging():
general_info['population_size'] = self.population_size
general_info['iterations'] = self.iterations
general_info['solution_length'] = self.solution_len
sync_publish('general_info', general_info)
for p in people:
self.population.append(p)
sync_publish('progressbar', {
'current': 100,
'estimated_remaining_seconds': 0
})
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def async_save() -> None:
CompletedTrade.insert(**d).execute()
if jh.is_debugging():
logger.info(f'Stored the completed trade record for {completed_trade.exchange}-{completed_trade.symbol}-{completed_trade.strategy_name} into database.')
def async_save():
DailyBalance.insert(**daily_balance).execute()
if jh.is_debugging():
logger.info(f'Stored daily portfolio balance record into the database: {daily_balance["asset"]} => {jh.format_currency(round(daily_balance["balance"], 2))}'
)
def async_save() -> None:
Order.insert(**d).execute()
if jh.is_debugging():
logger.info(f'Stored the executed order record for {order.exchange}-{order.symbol} into database.')
def simulator(candles, hyper_parameters=None):
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['trading_exchanges'][0],
config['app']['trading_symbols'][0])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
# convert DNS string into hyper_parameters
if r.dna and hyper_parameters is None:
hyper_parameters = jh.dna_to_hp(StrategyClass.hyper_parameters(),
r.dna)
r.strategy = StrategyClass()
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# init few objects that couldn't be initiated in Strategy __init__
r.strategy._init_objects()
# inject hyper parameters (used for optimize_mode)
if hyper_parameters is not None:
r.strategy.hp = hyper_parameters
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
_save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
_save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
# now that backtest is finished, add finishing balance
_save_daily_portfolio_balance()
def async_save():
DailyBalance.insert(**daily_balance).execute()
if jh.is_debugging():
logger.info('Stored daily portfolio balance record into the database: {} => {}'.format(
daily_balance['asset'], jh.format_currency(round(daily_balance['balance'], 2))
))
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError(
'Too many errors: less then half of the planned population size could be generated.'
)
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error('process failed - ID: {}'.format(
str(proc)))
logger.error("".join(
traceback.TracebackException.from_exception(
e).format()))
raise e
try:
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(
'a process exited with exitcode: {}'.
format(str(w.exitcode)))
except KeyboardInterrupt:
print(jh.color('Terminating session...', 'red'))
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [[
'Started At',
jh.get_arrow(self.start_time).humanize()
],
[
'Index/Total',
'{}/{}'.format((i + 1) * cores_num,
self.iterations)
],
[
'errors/info',
'{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
],
[
'Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]]
if jh.is_debugging():
table_items.insert(3, [
'Population Size, Solution Length',
'{}, {}'.format(self.population_size,
self.solution_len)
])
table.key_value(table_items,
'info',
alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
fittest_list = [
[
'Rank', 'DNA', 'Fitness',
'Training log || Testing log'
],
]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population
def simulator(candles: Dict[str, Dict[str, Union[str, np.ndarray]]],
hyperparameters=None) -> None:
begin_time_track = time.time()
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
first_candles_set = candles[key]['candles']
length = len(first_candles_set)
# to preset the array size for performance
store.app.starting_time = first_candles_set[0][0]
store.app.time = first_candles_set[0][0]
# initiate strategies
for r in router.routes:
StrategyClass = jh.get_strategy_class(r.strategy_name)
try:
r.strategy = StrategyClass()
except TypeError:
raise exceptions.InvalidStrategy(
"Looks like the structure of your strategy directory is incorrect. Make sure to include the strategy INSIDE the __init__.py file."
"\nIf you need working examples, check out: https://github.com/jesse-ai/example-strategies"
)
except:
raise
r.strategy.name = r.strategy_name
r.strategy.exchange = r.exchange
r.strategy.symbol = r.symbol
r.strategy.timeframe = r.timeframe
# inject hyper parameters (used for optimize_mode)
# convert DNS string into hyperparameters
if r.dna and hyperparameters is None:
hyperparameters = jh.dna_to_hp(r.strategy.hyperparameters(), r.dna)
# inject hyperparameters sent within the optimize mode
if hyperparameters is not None:
r.strategy.hp = hyperparameters
# init few objects that couldn't be initiated in Strategy __init__
# it also injects hyperparameters into self.hp in case the route does not uses any DNAs
r.strategy._init_objects()
selectors.get_position(r.exchange, r.symbol).strategy = r.strategy
# add initial balance
save_daily_portfolio_balance()
with click.progressbar(length=length,
label='Executing simulation...') as progressbar:
for i in range(length):
# update time
store.app.time = first_candles_set[i][0] + 60_000
# add candles
for j in candles:
short_candle = candles[j]['candles'][i]
if i != 0:
previous_short_candle = candles[j]['candles'][i - 1]
short_candle = _get_fixed_jumped_candle(
previous_short_candle, short_candle)
exchange = candles[j]['exchange']
symbol = candles[j]['symbol']
store.candles.add_candle(short_candle,
exchange,
symbol,
'1m',
with_execution=False,
with_generation=False)
# print short candle
if jh.is_debuggable('shorter_period_candles'):
print_candle(short_candle, True, symbol)
_simulate_price_change_effect(short_candle, exchange, symbol)
# generate and add candles for bigger timeframes
for timeframe in config['app']['considering_timeframes']:
# for 1m, no work is needed
if timeframe == '1m':
continue
count = jh.timeframe_to_one_minutes(timeframe)
until = count - ((i + 1) % count)
if (i + 1) % count == 0:
generated_candle = generate_candle_from_one_minutes(
timeframe,
candles[j]['candles'][(i - (count - 1)):(i + 1)])
store.candles.add_candle(generated_candle,
exchange,
symbol,
timeframe,
with_execution=False,
with_generation=False)
# update progressbar
if not jh.is_debugging() and not jh.should_execute_silently(
) and i % 60 == 0:
progressbar.update(60)
# now that all new generated candles are ready, execute
for r in router.routes:
count = jh.timeframe_to_one_minutes(r.timeframe)
# 1m timeframe
if r.timeframe == timeframes.MINUTE_1:
r.strategy._execute()
elif (i + 1) % count == 0:
# print candle
if jh.is_debuggable('trading_candles'):
print_candle(
store.candles.get_current_candle(
r.exchange, r.symbol, r.timeframe), False,
r.symbol)
r.strategy._execute()
# now check to see if there's any MARKET orders waiting to be executed
store.orders.execute_pending_market_orders()
if i != 0 and i % 1440 == 0:
save_daily_portfolio_balance()
if not jh.should_execute_silently():
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print('\n')
# print executed time for the backtest session
finish_time_track = time.time()
print(
'Executed backtest simulation in: ',
'{} seconds'.format(round(finish_time_track - begin_time_track,
2)))
for r in router.routes:
r.strategy._terminate()
store.orders.execute_pending_market_orders()
# now that backtest is finished, add finishing balance
save_daily_portfolio_balance()
def generate_initial_population(self):
"""
generates the initial population
"""
cores_num = cpu_count()
loop_length = int(self.population_size / cores_num)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna, dna_bucket):
fitness_score, fitness_log = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log))
try:
for _ in range(cores_num):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'log': d[2]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.get_arrow(self.start_time).humanize()],
['Index', '{}/{}'.format(len(self.population), self.population_size)],
['Trading Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe,
router.routes[0].strategy_name
)],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-'*10, '-'*10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))