def paper(debug: bool, dev: bool) -> None: """ trades in real-time on exchange with PAPER money """ validate_cwd() # set trading mode from jesse.config import config config['app']['trading_mode'] = 'papertrade' register_custom_exception_handler() # debug flag config['app']['debug_mode'] = debug from jesse_live import init from jesse.services.selectors import get_exchange live_config = locate('live-config.config') # validate that the "live-config.py" file exists if live_config is None: jh.error( 'You\'re either missing the live-config.py file or haven\'t logged in. Run "jesse login" to fix it.', True) jh.terminate_app() # inject live config init(config, live_config) # execute live session from jesse_live.live_mode import run run(dev)
def dd(msg: str) -> None: """ The dd function dumps the given variables and ends execution of the script. Used for debugging. :param msg: str """ print(msg) jh.terminate_app()
def terminate_session(): sync_publish('unexpectedTermination', { 'message': "Session terminated as the result of an uncaught exception", }) jesse_logger.error( 'Session terminated as the result of an uncaught exception') jh.terminate_app()
def initiate_ws(exchange_name: str) -> None: exchange_class = jh.get_config( f'app.live_drivers.{exchange_name}') try: self.drivers[exchange_name] = exchange_class() except TypeError: from jesse_live.info import SUPPORTED_EXCHANGES exchange_names = '' for se in SUPPORTED_EXCHANGES: exchange_names += '\n' + '"' + se['name'] + '"' error_msg = f'Driver for "{exchange_name}" is not supported yet. Supported exchanges are: {exchange_names}' jh.error(error_msg, force_print=True) jh.terminate_app()
def initiate_ws(exchange_name: str) -> None: from jesse_live.info import SUPPORTED_EXCHANGES, SUPPORTED_EXCHANGES_NAMES exchange_class = jh.get_config( f'app.live_drivers.{exchange_name}') if exchange_name not in SUPPORTED_EXCHANGES_NAMES: exchange_names = '' for se in SUPPORTED_EXCHANGES: exchange_names += f'\n "{se["name"]}"' error_msg = f'Driver for "{exchange_name}" is not supported yet. Supported exchanges are: {exchange_names}' jh.error(error_msg, force_print=True) jh.terminate_app() self.drivers[exchange_name] = exchange_class()
def run(self): try: mp.Process.run(self) except Exception as e: if type(e).__name__ == 'Termination': sync_publish('termination', {}) jh.terminate_app() else: sync_publish( 'exception', { 'error': f'{type(e).__name__}: {e}', 'traceback': str(traceback.format_exc()), }, ) print('Unhandled exception in the process:') print(traceback.format_exc()) terminate_session()
def handle_thread_exception(args) -> None: if args.exc_type == SystemExit: return if args.exc_type.__name__ == 'Termination': sync_publish('termination', {}) jh.terminate_app() else: # send notifications if it's a live session if jh.is_live(): jesse_logger.error( f'{args.exc_type.__name__}: {args.exc_value}') jesse_logger.info(str(traceback.format_exc())) sync_publish( 'exception', { 'error': f"{args.exc_type.__name__}: {str(args.exc_value)}", 'traceback': str(traceback.format_exc()) }) terminate_session()
def generate_initial_population(self) -> None: """ generates the initial population """ loop_length = int(self.population_size / self.cpu_cores) with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna: str, dna_bucket: list) -> None: try: fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing)) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'training_log': d[2], 'testing_log': d[3] }) # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started at', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index', f'{len(self.population)}/{self.population_size}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'], # TODO: add generated DNAs? # ['-'*10, '-'*10], # ['DNA', people[0]['dna']], # ['fitness', round(people[0]['fitness'], 6)], # ['training|testing logs', people[0]['log']], ] if jh.is_debugging(): table_items.insert(3, ['Population Size', self.population_size]) table_items.insert(3, ['Iterations', self.iterations]) table_items.insert(3, ['Solution Length', self.solution_len]) table_items.insert(3, ['-' * 10, '-' * 10]) table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') for p in people: self.population.append(p) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]: """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError('Too many errors: less then half of the planned population size could be generated.') # if even our best individual is too weak, then we better not continue if self.population[0]['fitness'] == 0.0001: print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. ' 'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow')) jh.terminate_app() loop_length = int(self.iterations / self.cpu_cores) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people: List) -> None: try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started At', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'] ] if jh.is_debugging(): table_items.insert( 3, ['Population Size, Solution Length', f'{self.population_size}, {self.solution_len}'] ) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals if jh.is_debugging(): fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ] else: fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError('self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%" if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][ 'PNL'] > 0 and self.population[j]['testing_log'][ 'PNL'] > 0: log = jh.style(log, 'bold') if jh.is_debugging(): fittest_list.append( [ j + 1, self.population[j]['dna'], self.population[j]['fitness'], log ], ) else: fittest_list.append( [ j + 1, self.population[j]['dna'], log ], ) if jh.is_debugging(): table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) else: table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(1, len(self.population) - 1) # never kill our best perforemr try: self.population[random_index] = baby except IndexError: print('=============') print(f'self.population_size: {self.population_size}') print(f'self.population length: {len(self.population)}') jh.terminate_app() self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print(f'fitness goal reached after iteration {i}') return baby # save progress after every n iterations if i != 0 and int(i * self.cpu_cores) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / self.cpu_cores) == 0: self.take_snapshot(i * self.cpu_cores) i += 1 print('\n\n') print(f'Finished {self.iterations} iterations.') return self.population
def evolve(self): """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() cores_num = cpu_count() loop_length = int(self.iterations / cores_num) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people): # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) for _ in range(cores_num): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() # update dashboard click.clear() progressbar.update(1) print('\n') table.key_value( [[ 'started at', jh.get_arrow(self.start_time).humanize() ], [ 'index/total', '{}/{}'.format( (i + 1) * cores_num, self.iterations) ], [ 'population_size, solution_len', '{}, {}'.format( self.population_size, self.solution_len) ], [ 'route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ]], 'info', alignments=('left', 'right')) print('\n') # print fittest individuals fittest_list = [ ['rank', 'DNA', 'fitness', 'training|testing logs'], ] if self.population_size > 50: number_of_ind_to_show = 25 elif self.population_size > 20: number_of_ind_to_show = 20 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError( 'self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): fittest_list.append([ j + 1, self.population[j]['dna'], self.population[j]['fitness'], self.population[j]['log'] ], ) table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(0, len(self.population) - 1) try: self.population[random_index] = baby except IndexError: print('=============') print('self.population_size: {}'.format( self.population_size)) print('self.population length: {}'.format( len(self.population))) jh.terminate_app() self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print('fitness goal reached after iteration {}'. format(i)) return baby # save progress after every n iterations if i != 0 and int(i * cores_num) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / cores_num) == 0: self.take_snapshot(i * cores_num) i += 1 print('\n\n') print('Finished {} iterations.'.format(self.iterations)) return self.population
# create and expose ENV_VALUES ENV_VALUES = dotenv_values('.env') if jh.is_unit_testing(): ENV_VALUES['POSTGRES_HOST'] = '127.0.0.1' ENV_VALUES['POSTGRES_NAME'] = 'jesse_db' ENV_VALUES['POSTGRES_PORT'] = '5432' ENV_VALUES['POSTGRES_USERNAME'] = '******' ENV_VALUES['POSTGRES_PASSWORD'] = '******' ENV_VALUES['REDIS_HOST'] = 'localhost' ENV_VALUES['REDIS_PORT'] = '6379' ENV_VALUES['REDIS_DB'] = 0 ENV_VALUES['REDIS_PASSWORD'] = '' # validation for existence of .env file if len(list(ENV_VALUES.keys())) == 0: jh.error( '.env file is missing from within your local project. ' 'This usually happens when you\'re in the wrong directory. ' '\n\nIf you haven\'t created a Jesse project yet, do that by running: \n' 'jesse make-project {name}\n' 'And then go into that project, and run the same command.', force_print=True) os._exit(1) jh.terminate_app() # raise FileNotFoundError('.env file is missing from within your local project. This usually happens when you\'re in the wrong directory. You can create one by running "cp .env.example .env"') if not jh.is_unit_testing() and ENV_VALUES['PASSWORD'] == '': raise EnvironmentError( 'You forgot to set the PASSWORD in your .env file')
def evolve(self): """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError( 'Too many errors: less then half of the planned population size could be generated.' ) cores_num = cpu_count() loop_length = int(self.iterations / cores_num) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people): try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error('process failed - ID: {}'.format( str(proc))) logger.error("".join( traceback.TracebackException.from_exception( e).format())) raise e try: for _ in range(cores_num): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error( 'a process exited with exitcode: {}'. format(str(w.exitcode))) except KeyboardInterrupt: print(jh.color('Terminating session...', 'red')) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [[ 'Started At', jh.get_arrow(self.start_time).humanize() ], [ 'Index/Total', '{}/{}'.format((i + 1) * cores_num, self.iterations) ], [ 'errors/info', '{}/{}'.format(len(store.logs.errors), len(store.logs.info)) ], [ 'Route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ]] if jh.is_debugging(): table_items.insert(3, [ 'Population Size, Solution Length', '{}, {}'.format(self.population_size, self.solution_len) ]) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals fittest_list = [ [ 'Rank', 'DNA', 'Fitness', 'Training log || Testing log' ], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError( 'self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): fittest_list.append([ j + 1, self.population[j]['dna'], self.population[j]['fitness'], self.population[j]['log'] ], ) table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(0, len(self.population) - 1) try: self.population[random_index] = baby except IndexError: print('=============') print('self.population_size: {}'.format( self.population_size)) print('self.population length: {}'.format( len(self.population))) jh.terminate_app() self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print('fitness goal reached after iteration {}'. format(i)) return baby # save progress after every n iterations if i != 0 and int(i * cores_num) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / cores_num) == 0: self.take_snapshot(i * cores_num) i += 1 print('\n\n') print('Finished {} iterations.'.format(self.iterations)) return self.population
def generate_initial_population(self): """ generates the initial population """ cores_num = cpu_count() loop_length = int(self.population_size / cores_num) with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna, dna_bucket): fitness_score, fitness_log = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log)) try: for _ in range(cores_num): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'log': d[2] }) # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started at', jh.get_arrow(self.start_time).humanize()], ['Index', '{}/{}'.format(len(self.population), self.population_size)], ['Trading Route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name )], # TODO: add generated DNAs? # ['-'*10, '-'*10], # ['DNA', people[0]['dna']], # ['fitness', round(people[0]['fitness'], 6)], # ['training|testing logs', people[0]['log']], ] if jh.is_debugging(): table_items.insert(3, ['Population Size', self.population_size]) table_items.insert(3, ['Iterations', self.iterations]) table_items.insert(3, ['Solution Length', self.solution_len]) table_items.insert(3, ['-'*10, '-'*10]) table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right')) for p in people: self.population.append(p) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))