def run(dna: bool = False) -> None: # trading routes arr = [] if not dna: print( jh.color('{}{}{}'.format('#' * 25, ' Trading Routes ', '#' * 25), 'blue') ) arr.append(('exchange', 'symbol', 'timeframe', 'strategy name', 'DNA')) else: print( jh.color('Translated DNAs into hyper-parameters:', 'blue') ) translated_DNAs_count = 0 for i, r in enumerate(router.routes): if dna and r.dna: translated_DNAs_count += 1 StrategyClass = jh.get_strategy_class(r.strategy_name) hyperparameters = jh.dna_to_hp(StrategyClass.hyperparameters(None), r.dna) table.key_value(hyperparameters.items(), r.strategy_name, uppercase_title=False) print('\n') else: arr.append([r.exchange, r.symbol, r.timeframe, r.strategy_name, r.dna]) if not dna: table.multi_value(arr) print('\n') else: if not translated_DNAs_count: print('No DNA string found.') # extra_candles if not dna: print( jh.color('{}{}{}'.format('#' * 25, ' Extra Candles ', '#' * 25), 'blue') ) arr = [('exchange', 'symbol', 'timeframe')] for i, r in enumerate(router.extra_candles): arr.append([r[0], r[1], r[2]]) table.multi_value(arr) print('\n')
def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = '{}-{}'.format(config['app']['considering_candles'][0][0], config['app']['considering_candles'][0][1]) table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', str(round(np.average(change), 2)) + "%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs store_logs(json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns() # QuantStats' report if full_reports: quantstats.quantstats_tearsheet() else: print(jh.color('No trades were made.', 'yellow'))
def generate_initial_population(self) -> None: """ generates the initial population """ loop_length = int(self.population_size / self.cpu_cores) with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna: str, dna_bucket: list) -> None: try: fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing)) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'training_log': d[2], 'testing_log': d[3] }) # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started at', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index', f'{len(self.population)}/{self.population_size}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'], # TODO: add generated DNAs? # ['-'*10, '-'*10], # ['DNA', people[0]['dna']], # ['fitness', round(people[0]['fitness'], 6)], # ['training|testing logs', people[0]['log']], ] if jh.is_debugging(): table_items.insert(3, ['Population Size', self.population_size]) table_items.insert(3, ['Iterations', self.iterations]) table_items.insert(3, ['Solution Length', self.solution_len]) table_items.insert(3, ['-' * 10, '-' * 10]) table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') for p in people: self.population.append(p) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]: """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError('Too many errors: less then half of the planned population size could be generated.') # if even our best individual is too weak, then we better not continue if self.population[0]['fitness'] == 0.0001: print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. ' 'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow')) jh.terminate_app() loop_length = int(self.iterations / self.cpu_cores) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people: List) -> None: try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started At', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'] ] if jh.is_debugging(): table_items.insert( 3, ['Population Size, Solution Length', f'{self.population_size}, {self.solution_len}'] ) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals if jh.is_debugging(): fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ] else: fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError('self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%" if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][ 'PNL'] > 0 and self.population[j]['testing_log'][ 'PNL'] > 0: log = jh.style(log, 'bold') if jh.is_debugging(): fittest_list.append( [ j + 1, self.population[j]['dna'], self.population[j]['fitness'], log ], ) else: fittest_list.append( [ j + 1, self.population[j]['dna'], log ], ) if jh.is_debugging(): table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) else: table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(1, len(self.population) - 1) # never kill our best perforemr try: self.population[random_index] = baby except IndexError: print('=============') print(f'self.population_size: {self.population_size}') print(f'self.population length: {len(self.population)}') jh.terminate_app() self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print(f'fitness goal reached after iteration {i}') return baby # save progress after every n iterations if i != 0 and int(i * self.cpu_cores) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / self.cpu_cores) == 0: self.take_snapshot(i * self.cpu_cores) i += 1 print('\n\n') print(f'Finished {self.iterations} iterations.') return self.population
def generate_initial_population(self): """ generates the initial population """ cores_num = cpu_count() loop_length = int(self.population_size / cores_num) with click.progressbar( length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna, dna_bucket): fitness_score, fitness_log = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log)) for _ in range(cores_num): dna = ''.join( choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'log': d[2] }) # update dashboard click.clear() progressbar.update(1) print('\n') table.key_value([ ['started at', jh.get_arrow(self.start_time).humanize()], [ 'index/total', '{}/{}'.format(len(self.population), self.population_size) ], ['-', '-'], ['population_size', self.population_size], ['iterations', self.iterations], ['solution_len', self.solution_len], [ 'route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ], ['-', '-'], ['DNA', people[0]['dna']], ['fitness', round(people[0]['fitness'], 6)], ['training|testing logs', people[0]['log']], ], 'baby', alignments=('left', 'right')) for p in people: self.population.append(p) # sort the population self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self): """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() cores_num = cpu_count() loop_length = int(self.iterations / cores_num) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people): # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) for _ in range(cores_num): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() # update dashboard click.clear() progressbar.update(1) print('\n') table.key_value( [[ 'started at', jh.get_arrow(self.start_time).humanize() ], [ 'index/total', '{}/{}'.format( (i + 1) * cores_num, self.iterations) ], [ 'population_size, solution_len', '{}, {}'.format( self.population_size, self.solution_len) ], [ 'route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ]], 'info', alignments=('left', 'right')) print('\n') # print fittest individuals fittest_list = [ ['rank', 'DNA', 'fitness', 'training|testing logs'], ] if self.population_size > 50: number_of_ind_to_show = 25 elif self.population_size > 20: number_of_ind_to_show = 20 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError( 'self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): fittest_list.append([ j + 1, self.population[j]['dna'], self.population[j]['fitness'], self.population[j]['log'] ], ) table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(0, len(self.population) - 1) try: self.population[random_index] = baby except IndexError: print('=============') print('self.population_size: {}'.format( self.population_size)) print('self.population length: {}'.format( len(self.population))) jh.terminate_app() self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print('fitness goal reached after iteration {}'. format(i)) return baby # save progress after every n iterations if i != 0 and int(i * cores_num) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / cores_num) == 0: self.take_snapshot(i * cores_num) i += 1 print('\n\n') print('Finished {} iterations.'.format(self.iterations)) return self.population
def run(start_date: str, finish_date: str, candles=None, chart=False, tradingview=False, csv=False, json=False): # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = '{}-{}'.format(config['app']['trading_exchanges'][0], config['app']['trading_symbols'][0]) table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades statistics if store.completed_trades.count > 0: print('\n') table.key_value(report.portfolio_metrics(), 'Metrics', alignments=('left', 'right')) print('\n') # save logs store_logs(json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns() else: print(jh.color('No trades were made.', 'yellow'))
def run(start_date: str, finish_date: str, candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None, chart: bool = False, tradingview: bool = False, full_reports: bool = False, csv: bool = False, json: bool = False) -> None: # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = f"{config['app']['considering_candles'][0][0]}-{config['app']['considering_candles'][0][1]}" table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation simulator(candles) if not jh.should_execute_silently(): # print trades metrics if store.completed_trades.count > 0: change = [] # calcualte market change for e in router.routes: if e.strategy is None: return first = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(start_date), Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() last = Candle.select(Candle.close).where( Candle.timestamp == jh.date_to_timestamp(finish_date) - 60000, Candle.exchange == e.exchange, Candle.symbol == e.symbol).first() change.append( ((last.close - first.close) / first.close) * 100.0) data = report.portfolio_metrics() data.append( ['Market Change', f"{str(round(np.average(change), 2))}%"]) print('\n') table.key_value(data, 'Metrics', alignments=('left', 'right')) print('\n') # save logs more = "" routes_count = len(router.routes) if routes_count > 1: more = f"-and-{routes_count-1}-more" study_name = f"{router.routes[0].strategy_name}-{router.routes[0].exchange}-{router.routes[0].symbol}-{router.routes[0].timeframe}{more}-{start_date}-{finish_date}" store_logs(study_name, json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns(study_name) # QuantStats' report if full_reports: price_data = [] # load close candles for Buy and hold and calculate pct_change for index, c in enumerate( config['app']['considering_candles']): exchange, symbol = c[0], c[1] if exchange in config['app'][ 'trading_exchanges'] and symbol in config['app'][ 'trading_symbols']: # fetch from database candles_tuple = Candle.select( Candle.timestamp, Candle.close).where( Candle.timestamp.between( jh.date_to_timestamp(start_date), jh.date_to_timestamp(finish_date) - 60000), Candle.exchange == exchange, Candle.symbol == symbol).order_by( Candle.timestamp.asc()).tuples() candles = np.array(candles_tuple) timestamps = candles[:, 0] price_data.append(candles[:, 1]) price_data = np.transpose(price_data) price_df = pd.DataFrame(price_data, index=pd.to_datetime(timestamps, unit="ms"), dtype=float).resample('D').mean() price_pct_change = price_df.pct_change(1).fillna(0) bh_daily_returns_all_routes = price_pct_change.mean(1) quantstats.quantstats_tearsheet(bh_daily_returns_all_routes, study_name) else: print(jh.color('No trades were made.', 'yellow'))
def generate_signal(start_date: str, finish_date: str, candles=None, chart=False, tradingview=False, csv=False, json=False): # clear the screen if not jh.should_execute_silently(): click.clear() # validate routes validate_routes(router) # initiate candle store store.candles.init_storage(5000) # load historical candles if candles is None: print('loading candles...') candles = load_candles(start_date, finish_date) click.clear() if not jh.should_execute_silently(): # print candles table key = '{}-{}'.format(config['app']['considering_candles'][0][0], config['app']['considering_candles'][0][1]) table.key_value(stats.candles(candles[key]['candles']), 'candles', alignments=('left', 'right')) print('\n') # print routes table table.multi_value(stats.routes(router.routes)) print('\n') # print guidance for debugging candles if jh.is_debuggable('trading_candles') or jh.is_debuggable( 'shorter_period_candles'): print( ' Symbol | timestamp | open | close | high | low | volume' ) # run backtest simulation signal_simulator(candles) if not jh.should_execute_silently(): # print trades statistics if store.completed_trades.count > 0: print('\n') table.key_value(report.portfolio_metrics(), 'Metrics', alignments=('left', 'right')) print('\n') # save logs store_logs(json, tradingview, csv) if chart: charts.portfolio_vs_asset_returns() # # clone the trades so that the original is not mutated # completed_trades = copy.deepcopy(store.completed_trades.trades) # # for trade in completed_trades: # print(trade.to_dict()) # # # filter trades which were generated for current day # completed_trades = filter(lambda x: x.opened_at == jh.get_current_time_in_epoch(), completed_trades) # # # TODO: add a slack notification here instead of print to notify the user # print("Trades to execute today: ") # for trade in completed_trades: # print(trade.to_dict()) else: print(jh.color('No trades were completed/closed.', 'yellow')) open_positions = store.positions.get_open_positions() open_orders = store.orders.get_all_active_orders() current_report = jh.generate_signals_report(open_positions, open_orders) return current_report
def evolve(self): """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError( 'Too many errors: less then half of the planned population size could be generated.' ) cores_num = cpu_count() loop_length = int(self.iterations / cores_num) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people): try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error('process failed - ID: {}'.format( str(proc))) logger.error("".join( traceback.TracebackException.from_exception( e).format())) raise e try: for _ in range(cores_num): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error( 'a process exited with exitcode: {}'. format(str(w.exitcode))) except KeyboardInterrupt: print(jh.color('Terminating session...', 'red')) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [[ 'Started At', jh.get_arrow(self.start_time).humanize() ], [ 'Index/Total', '{}/{}'.format((i + 1) * cores_num, self.iterations) ], [ 'errors/info', '{}/{}'.format(len(store.logs.errors), len(store.logs.info)) ], [ 'Route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name) ]] if jh.is_debugging(): table_items.insert(3, [ 'Population Size, Solution Length', '{}, {}'.format(self.population_size, self.solution_len) ]) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals fittest_list = [ [ 'Rank', 'DNA', 'Fitness', 'Training log || Testing log' ], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError( 'self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): fittest_list.append([ j + 1, self.population[j]['dna'], self.population[j]['fitness'], self.population[j]['log'] ], ) table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(0, len(self.population) - 1) try: self.population[random_index] = baby except IndexError: print('=============') print('self.population_size: {}'.format( self.population_size)) print('self.population length: {}'.format( len(self.population))) jh.terminate_app() self.population = list( sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print('fitness goal reached after iteration {}'. format(i)) return baby # save progress after every n iterations if i != 0 and int(i * cores_num) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / cores_num) == 0: self.take_snapshot(i * cores_num) i += 1 print('\n\n') print('Finished {} iterations.'.format(self.iterations)) return self.population
def generate_initial_population(self): """ generates the initial population """ cores_num = cpu_count() loop_length = int(self.population_size / cores_num) with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar: for i in range(loop_length): people = [] with Manager() as manager: dna_bucket = manager.list([]) workers = [] def get_fitness(dna, dna_bucket): fitness_score, fitness_log = self.fitness(dna) dna_bucket.append((dna, fitness_score, fitness_log)) try: for _ in range(cores_num): dna = ''.join(choices(self.charset, k=self.solution_len)) w = Process(target=get_fitness, args=(dna, dna_bucket)) w.start() workers.append(w) # join workers for w in workers: w.join() except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise for d in dna_bucket: people.append({ 'dna': d[0], 'fitness': d[1], 'log': d[2] }) # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started at', jh.get_arrow(self.start_time).humanize()], ['Index', '{}/{}'.format(len(self.population), self.population_size)], ['Trading Route', '{}, {}, {}, {}'.format( router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe, router.routes[0].strategy_name )], # TODO: add generated DNAs? # ['-'*10, '-'*10], # ['DNA', people[0]['dna']], # ['fitness', round(people[0]['fitness'], 6)], # ['training|testing logs', people[0]['log']], ] if jh.is_debugging(): table_items.insert(3, ['Population Size', self.population_size]) table_items.insert(3, ['Iterations', self.iterations]) table_items.insert(3, ['Solution Length', self.solution_len]) table_items.insert(3, ['-'*10, '-'*10]) table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right')) for p in people: self.population.append(p) # sort the population self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))