def handle_thread_exception(args): """ :param args: :return: """ if args.exc_type == SystemExit: return # handle Breaking exceptions if args.exc_type in [ exceptions.ConfigException, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color( '{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow')) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error('{}: {}'.format(args.exc_type.__name__, args.exc_value)) if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(args.exc_type, args.exc_value, args.exc_traceback)) else: print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(args.exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color( '{}: {}'.format(args.exc_type.__name__, args.exc_value), 'yellow')) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/paper-trade.txt'), 'red')) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/live-trade.txt'), 'red')) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}' .format('storage/logs/collect.txt'), 'red'))
def handle_exception(exc_type, exc_value, exc_traceback) -> None: if issubclass(exc_type, KeyboardInterrupt): sys.excepthook(exc_type, exc_value, exc_traceback) return # handle Breaking exceptions if exc_type in [ exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow') ) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error( f'{exc_type.__name__}: {exc_value}' ) if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback)) else: print(f"{'=' * 30} EXCEPTION TRACEBACK:") traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow') ) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt', 'red' ) ) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt', 'red' ) ) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt', 'red' ) )
def log(self, msg, log_type='info'): msg = str(msg) if log_type == 'info': logger.info(msg) elif log_type == 'error': logger.error(msg) else: raise ValueError(f'log_type should be either "info" or "error". You passed {log_type}')
def get_fitness(dna: str, dna_bucket: list) -> None: try: fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing)) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e
def error(msg: str, force_print: bool = False) -> None: # send notifications if it's a live session if is_live(): from jesse.services import logger logger.error(msg) if force_print: print(color(msg, 'red')) else: print(color(msg, 'red'))
def terminate_session(): sync_publish('unexpectedTermination', { 'message': "Session terminated as the result of an uncaught exception", }) jesse_logger.error( 'Session terminated as the result of an uncaught exception') jh.terminate_app()
def get_baby(people: List) -> None: try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e
def get_fitness(dna, dna_bucket): try: fitness_score, fitness_log = self.fitness(dna) dna_bucket.append( (dna, fitness_score, fitness_log)) except Exception as e: proc = os.getpid() logger.error('process failed - ID: {}'.format( str(proc))) logger.error("".join( traceback.TracebackException.from_exception( e).format())) raise e
def test_can_log_error_by_firing_event(): set_up() # fire first error event logger.error('first error!!!!!') first_logged_error = {'time': jh.now(), 'message': 'first error!!!!!'} assert store.logs.errors == [first_logged_error] # fire second error event logger.error('second error!!!!!') second_logged_error = {'time': jh.now(), 'message': 'second error!!!!!'} assert store.logs.errors == [first_logged_error, second_logged_error]
def log(msg: str, log_type: str = 'info') -> None: msg = str(msg) if log_type == 'info': logger.info(msg) if jh.is_live(): notifier.notify(msg) elif log_type == 'error': logger.error(msg) if jh.is_live(): notifier.notify(msg) notifier.notify_urgently(msg) else: raise ValueError(f'log_type should be either "info" or "error". You passed {log_type}')
def get_fitness(dna: str, dna_bucket: list) -> None: try: # check if the DNA is already in the list if all(dna_tuple[0] != dna for dna_tuple in dna_bucket): fitness_score, fitness_log_training, fitness_log_testing = self.fitness( dna) dna_bucket.append( (dna, fitness_score, fitness_log_training, fitness_log_testing)) else: raise ValueError( f"Initial Population: Double DNA: {dna}") except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join( traceback.TracebackException.from_exception( e).format())) raise e
def handle_thread_exception(args) -> None: if args.exc_type == SystemExit: return if args.exc_type.__name__ == 'Termination': sync_publish('termination', {}) jh.terminate_app() else: # send notifications if it's a live session if jh.is_live(): jesse_logger.error( f'{args.exc_type.__name__}: {args.exc_value}') jesse_logger.info(str(traceback.format_exc())) sync_publish( 'exception', { 'error': f"{args.exc_type.__name__}: {str(args.exc_value)}", 'traceback': str(traceback.format_exc()) }) terminate_session()
def generate_initial_population(self) -> None: """ generates the initial population """ loop_length = int(self.population_size / self.cpu_cores) with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna: str, dna_bucket: list) -> None: try: fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing)) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'training_log': d[2], 'testing_log': d[3] }) # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started at', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index', f'{len(self.population)}/{self.population_size}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'], # TODO: add generated DNAs? # ['-'*10, '-'*10], # ['DNA', people[0]['dna']], # ['fitness', round(people[0]['fitness'], 6)], # ['training|testing logs', people[0]['log']], ] if jh.is_debugging(): table_items.insert(3, ['Population Size', self.population_size]) table_items.insert(3, ['Iterations', self.iterations]) table_items.insert(3, ['Solution Length', self.solution_len]) table_items.insert(3, ['-' * 10, '-' * 10]) table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') for p in people: self.population.append(p) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]: """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError('Too many errors: less then half of the planned population size could be generated.') # if even our best individual is too weak, then we better not continue if self.population[0]['fitness'] == 0.0001: print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. ' 'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow')) jh.terminate_app() loop_length = int(self.iterations / self.cpu_cores) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people: List) -> None: try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started At', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'] ] if jh.is_debugging(): table_items.insert( 3, ['Population Size, Solution Length', f'{self.population_size}, {self.solution_len}'] ) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals if jh.is_debugging(): fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ] else: fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError('self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%" if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][ 'PNL'] > 0 and self.population[j]['testing_log'][ 'PNL'] > 0: log = jh.style(log, 'bold') if jh.is_debugging(): fittest_list.append( [ j + 1, self.population[j]['dna'], self.population[j]['fitness'], log ], ) else: fittest_list.append( [ j + 1, self.population[j]['dna'], log ], ) if jh.is_debugging(): table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) else: table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(1, len(self.population) - 1) # never kill our best perforemr try: self.population[random_index] = baby except IndexError: print('=============') print(f'self.population_size: {self.population_size}') print(f'self.population length: {len(self.population)}') jh.terminate_app() self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print(f'fitness goal reached after iteration {i}') return baby # save progress after every n iterations if i != 0 and int(i * self.cpu_cores) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / self.cpu_cores) == 0: self.take_snapshot(i * self.cpu_cores) i += 1 print('\n\n') print(f'Finished {self.iterations} iterations.') return self.population
def evolve(self) -> list: """ the main method, that runs the evolutionary algorithm """ # clear the logs to start from a clean slate jh.clear_file('storage/logs/optimize-mode.txt') logger.log_optimize_mode('Optimization session started') if self.started_index == 0: logger.log_optimize_mode( f"Generating {self.population_size} population size (random DNAs) using {self.cpu_cores} CPU cores" ) self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: msg = f'Too many errors! less than half of the expected population size could be generated. Only {len(self.population)} indviduals from planned {self.population_size} are usable.' logger.log_optimize_mode(msg) raise ValueError(msg) # if even our best individual is too weak, then we better not continue if self.population[0]['fitness'] == 0.0001: msg = 'Cannot continue because no individual with the minimum fitness-score was found. Your strategy seems to be flawed or maybe it requires modifications. ' logger.log_optimize_mode(msg) raise exceptions.InvalidStrategy(msg) loop_length = int(self.iterations / self.cpu_cores) i = self.started_index progressbar = Progressbar(loop_length) while i < loop_length: with Manager() as manager: people_bucket = manager.list([]) workers = [] try: for _ in range(self.cpu_cores): mommy = self.select_person() daddy = self.select_person() w = Process( target=create_baby, args=( people_bucket, mommy, daddy, self.solution_len, self.charset, jh.get_config('env.optimization'), router.formatted_routes, router.formatted_extra_routes, self.strategy_hp, self.training_candles, self.testing_candles, self.optimal_total ) ) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {w.exitcode}') except exceptions.Termination: self._handle_termination(manager, workers) # update dashboard click.clear() self.update_progressbar(progressbar) # general_info streams general_info = { 'started_at': jh.timestamp_to_arrow(self.start_time).humanize(), 'index': f'{(i + 1) * self.cpu_cores}/{self.iterations}', 'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}', 'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}', 'average_execution_seconds': self.average_execution_seconds } if jh.is_debugging(): general_info['population_size'] = self.population_size general_info['iterations'] = self.iterations general_info['solution_length'] = self.solution_len sync_publish('general_info', general_info) if self.population_size > 50: number_of_ind_to_show = 40 elif self.population_size > 20: number_of_ind_to_show = 15 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError('self.population_size cannot be less than 10') best_candidates = [{ 'rank': j + 1, 'dna': self.population[j]['dna'], 'fitness': round(self.population[j]['fitness'], 4), 'training_win_rate': self.population[j]['training_log']['win-rate'], 'training_total_trades': self.population[j]['training_log']['total'], 'training_pnl': self.population[j]['training_log']['PNL'], 'testing_win_rate': self.population[j]['testing_log']['win-rate'], 'testing_total_trades': self.population[j]['testing_log']['total'], 'testing_pnl': self.population[j]['testing_log']['PNL'], } for j in range(number_of_ind_to_show)] sync_publish('best_candidates', best_candidates) # one person has to die and be replaced with the newborn baby for baby in people_bucket: # never kill our best performer random_index = randint(1, len(self.population) - 1) self.population[random_index] = baby self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: self.update_progressbar(progressbar, finished=True) sync_publish('alert', { 'message': f'Fitness goal reached after iteration {i*self.cpu_cores}', 'type': 'success' }) return baby # TODO: bring back progress resumption # # save progress after every n iterations # if i != 0 and int(i * self.cpu_cores) % 50 == 0: # self.save_progress(i) # TODO: bring back # # store a take_snapshot of the fittest individuals of the population # if i != 0 and i % int(100 / self.cpu_cores) == 0: # self.take_snapshot(i * self.cpu_cores) i += 1 sync_publish('alert', { 'message': f"Finished {self.iterations} iterations. Check your best DNA candidates, " f"if you don't like any of them, try modifying your strategy.", 'type': 'success' }) return self.population
def generate_initial_population(self) -> None: """ generates the initial population """ length = int(self.population_size / self.cpu_cores) progressbar = Progressbar(length) for i in range(length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] try: for _ in range(self.cpu_cores): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process( target=get_and_add_fitness_to_the_bucket, args=( dna_bucket, jh.get_config('env.optimization'), router.formatted_routes, router.formatted_extra_routes, self.strategy_hp, dna, self.training_candles, self.testing_candles, self.optimal_total ) ) w.start() workers.append(w) # join workers for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {w.exitcode}') except exceptions.Termination: self._handle_termination(manager, workers) for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'training_log': d[2], 'testing_log': d[3] }) # update dashboard self.update_progressbar(progressbar) # general_info streams general_info = { 'started_at': jh.timestamp_to_arrow(self.start_time).humanize(), 'index': f'{(i + 1) * self.cpu_cores}/{self.population_size}', 'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}', 'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}', 'average_execution_seconds': self.average_execution_seconds } if jh.is_debugging(): general_info['population_size'] = self.population_size general_info['iterations'] = self.iterations general_info['solution_length'] = self.solution_len sync_publish('general_info', general_info) for p in people: self.population.append(p) sync_publish('progressbar', { 'current': 100, 'estimated_remaining_seconds': 0 }) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self): """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError( 'Too many errors: less then half of the planned population size could be generated.' ) cores_num = cpu_count() loop_length = int(self.iterations / cores_num) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people): try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error('process failed - ID: {}'.format( str(proc))) logger.error("".join( traceback.TracebackException.from_exception( e).format())) raise e try: for _ in range(cores_num): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error( 'a process exited with exitcode: {}'. format(str(w.exitcode))) except KeyboardInterrupt: print(jh.color('Terminating session...', 'red')) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [[ 'Started At', jh.get_arrow(self.start_time).humanize() ], [ 'Index/Total', '{}/{}'.format((i + 1) * cores_num, self.iterations) ], [ 'errors/info', '{}/{}'.format(len(store.logs.errors), len(store.logs.info)) ], [ 'Route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ]] if jh.is_debugging(): table_items.insert(3, [ 'Population Size, Solution Length', '{}, {}'.format(self.population_size, self.solution_len) ]) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals fittest_list = [ [ 'Rank', 'DNA', 'Fitness', 'Training log || Testing log' ], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError( 'self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): fittest_list.append([ j + 1, self.population[j]['dna'], self.population[j]['fitness'], self.population[j]['log'] ], ) table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(0, len(self.population) - 1) try: self.population[random_index] = baby except IndexError: print('=============') print('self.population_size: {}'.format( self.population_size)) print('self.population length: {}'.format( len(self.population))) jh.terminate_app() self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print('fitness goal reached after iteration {}'. format(i)) return baby # save progress after every n iterations if i != 0 and int(i * cores_num) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / cores_num) == 0: self.take_snapshot(i * cores_num) i += 1 print('\n\n') print('Finished {} iterations.'.format(self.iterations)) return self.population