def candles(): """ :return: """ array = [] candle_keys = [] # add routes for e in router.routes: if e.strategy is None: return candle_keys.append({ 'exchange': e.exchange, 'symbol': e.symbol, 'timeframe': e.timeframe }) # add extra_routes for e in router.extra_candles: candle_keys.append({ 'exchange': e[0], 'symbol': e[1], 'timeframe': e[2] }) # headers array.append([ 'exchange-symbol-timeframe', 'timestamp', 'open', 'close', 'high', 'low' ]) for k in candle_keys: try: current_candle = store.candles.get_current_candle( k['exchange'], k['symbol'], k['timeframe']) green = is_bullish(current_candle) bold = k['symbol'] in config['app']['trading_symbols'] and k[ 'timeframe'] in config['app']['trading_timeframes'] key = jh.key(k['exchange'], k['symbol'], k['timeframe']) array.append([ jh.style(key, 'underline' if bold else None), jh.color(jh.timestamp_to_time(current_candle[0]), 'green' if green else 'red'), jh.color(str(current_candle[1]), 'green' if green else 'red'), jh.color(str(current_candle[2]), 'green' if green else 'red'), jh.color(str(current_candle[3]), 'green' if green else 'red'), jh.color(str(current_candle[4]), 'green' if green else 'red'), ]) except IndexError: return except Exception: raise return array
def test_style(): assert jh.style('test', 'bold') == "\x1b[1mtest\x1b[0m" assert jh.style('test', 'u') == "\x1b[4mtest\x1b[0m"
def fitness(self, dna) -> tuple: hp = jh.dna_to_hp(self.strategy_hp, dna) # init candle store store.candles.init_storage(5000) # inject required TRAINING candles to the candle store for num, c in enumerate(config['app']['considering_candles']): required_candles.inject_required_candles_to_store( self.training_initial_candles[num], c[0], c[1]) # run backtest simulation simulator(self.training_candles, hp) log = '' # TODO: some of these have to be dynamic based on how many days it's trading for like for example "total" # I'm guessing we should accept "optimal" total from command line if store.completed_trades.count > 5: training_data = stats.trades(store.completed_trades.trades, store.app.daily_balance) total_effect_rate = log10(training_data['total']) / log10( self.optimal_total) if total_effect_rate > 1: total_effect_rate = 1 win_rate = training_data['win_rate'] ratio_config = jh.get_config('env.optimization.ratio', 'sharpe') if ratio_config == 'sharpe': ratio = training_data['sharpe_ratio'] elif ratio_config == 'calmar': ratio = training_data['calmar_ratio'] elif ratio_config == 'sortiono': ratio = training_data['sortino_ratio'] elif ratio_config == 'omega': ratio = training_data['omega_ratio'] else: raise ValueError( 'The entered ratio configuration `{}` for the optimization is unknown. Choose between sharpe, calmar, sortino and omega.' .format(ratio_config)) if ratio < 0: score = 0.0001 # reset store store.reset() return score, log ratio_normalized = jh.normalize(ratio, -.5, 4) # log for debugging/monitoring log = 'win-rate: {}%, total: {}, PNL: {}%'.format( int(win_rate * 100), training_data['total'], round(training_data['net_profit_percentage'], 2), ) score = total_effect_rate * ratio_normalized # perform backtest with testing data. this is using data # model hasn't trained for. if it works well, there is # high change it will do good with future data too. store.reset() store.candles.init_storage(5000) # inject required TESTING candles to the candle store for num, c in enumerate(config['app']['considering_candles']): required_candles.inject_required_candles_to_store( self.testing_initial_candles[num], c[0], c[1]) # run backtest simulation simulator(self.testing_candles, hp) testing_data = stats.trades(store.completed_trades.trades, store.app.daily_balance) # log for debugging/monitoring log += ' || ' if store.completed_trades.count > 0: log += 'win-rate: {}%, total: {}, PNL: {}%'.format( int(testing_data['win_rate'] * 100), testing_data['total'], round(testing_data['net_profit_percentage'], 2), ) if testing_data['net_profit_percentage'] > 0 and training_data[ 'net_profit_percentage'] > 0: log = jh.style(log, 'bold') else: log += 'win-rate: -, total: -, PNL%: -' else: score = 0.0001 # reset store store.reset() return score, log
def evolve(self) -> List[Any]: """ the main method, that runs the evolutionary algorithm """ # generate the population if starting if self.started_index == 0: self.generate_initial_population() if len(self.population) < 0.5 * self.population_size: raise ValueError('Too many errors: less then half of the planned population size could be generated.') # if even our best individual is too weak, then we better not continue if self.population[0]['fitness'] == 0.0001: print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. ' 'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow')) jh.terminate_app() loop_length = int(self.iterations / self.cpu_cores) i = self.started_index with click.progressbar(length=loop_length, label='Evolving...') as progressbar: while i < loop_length: with Manager() as manager: people = manager.list([]) workers = [] def get_baby(people: List) -> None: try: # let's make a baby together LOL baby = self.make_love() # let's mutate baby's genes, who knows, maybe we create a x-man or something baby = self.mutate(baby) people.append(baby) except Exception as e: proc = os.getpid() logger.error(f'process failed - ID: {str(proc)}') logger.error("".join(traceback.TracebackException.from_exception(e).format())) raise e try: for _ in range(self.cpu_cores): w = Process(target=get_baby, args=[people]) w.start() workers.append(w) for w in workers: w.join() if w.exitcode > 0: logger.error(f'a process exited with exitcode: {str(w.exitcode)}') except KeyboardInterrupt: print( jh.color('Terminating session...', 'red') ) # terminate all workers for w in workers: w.terminate() # shutdown the manager process manually since garbage collection cannot won't get to do it for us manager.shutdown() # now we can terminate the main session safely jh.terminate_app() except: raise # update dashboard click.clear() progressbar.update(1) print('\n') table_items = [ ['Started At', jh.timestamp_to_arrow(self.start_time).humanize()], ['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'], ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'], ['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'] ] if jh.is_debugging(): table_items.insert( 3, ['Population Size, Solution Length', f'{self.population_size}, {self.solution_len}'] ) table.key_value(table_items, 'info', alignments=('left', 'right')) # errors if jh.is_debugging() and len(report.errors()): print('\n') table.key_value(report.errors(), 'Error Logs') print('\n') print('Best DNA candidates:') print('\n') # print fittest individuals if jh.is_debugging(): fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ] else: fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ] if self.population_size > 50: number_of_ind_to_show = 15 elif self.population_size > 20: number_of_ind_to_show = 10 elif self.population_size > 9: number_of_ind_to_show = 9 else: raise ValueError('self.population_size cannot be less than 10') for j in range(number_of_ind_to_show): log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%" if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][ 'PNL'] > 0 and self.population[j]['testing_log'][ 'PNL'] > 0: log = jh.style(log, 'bold') if jh.is_debugging(): fittest_list.append( [ j + 1, self.population[j]['dna'], self.population[j]['fitness'], log ], ) else: fittest_list.append( [ j + 1, self.population[j]['dna'], log ], ) if jh.is_debugging(): table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left')) else: table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left')) # one person has to die and be replaced with the newborn baby for baby in people: random_index = randint(1, len(self.population) - 1) # never kill our best perforemr try: self.population[random_index] = baby except IndexError: print('=============') print(f'self.population_size: {self.population_size}') print(f'self.population length: {len(self.population)}') jh.terminate_app() self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True)) # reaching the fitness goal could also end the process if baby['fitness'] >= self.fitness_goal: progressbar.update(self.iterations - i) print('\n') print(f'fitness goal reached after iteration {i}') return baby # save progress after every n iterations if i != 0 and int(i * self.cpu_cores) % 50 == 0: self.save_progress(i) # store a take_snapshot of the fittest individuals of the population if i != 0 and i % int(100 / self.cpu_cores) == 0: self.take_snapshot(i * self.cpu_cores) i += 1 print('\n\n') print(f'Finished {self.iterations} iterations.') return self.population
def fitness(self, dna) -> tuple: hp = jh.dna_to_hp(self.strategy_hp, dna) # init candle store store.candles.init_storage(5000) # inject required TRAINING candles to the candle store required_candles.inject_required_candles_to_store( self.required_initial_training_candles, self.exchange, self.symbol) # run backtest simulation simulator(self.training_candles, hp) log = '' # TODO: some of these have to be dynamic based on how many days it's trading for like for example "total" # I'm guessing we should accept "optimal" total from command line if store.completed_trades.count > 5: training_data = stats.trades(store.completed_trades.trades) optimal_expected_total = 100 total = jh.normalize(training_data['total'], 0, 200) total_effect_rate = log10( training_data['total']) / log10(optimal_expected_total) win_rate = training_data['win_rate'] # log for debugging/monitoring log = 'win_rate:[{}-{}], total:[{}-{}], PNL%:[{}], TER:[{}]'.format( round(win_rate, 2), round(training_data['win_rate'], 2), round(total, 2), training_data['total'], round(training_data['pnl_percentage'], 2), round(total_effect_rate, 3)) # the fitness score score = win_rate * total_effect_rate # perform backtest with testing data. this is using data # model hasn't trained for. if it works well, there is # high change it will do good with future data too. store.reset() store.candles.init_storage(5000) # inject required TESTING candles to the candle store required_candles.inject_required_candles_to_store( self.required_initial_testing_candles, self.exchange, self.symbol) # run backtest simulation simulator(self.testing_candles, hp) testing_data = stats.trades(store.completed_trades.trades) # log for debugging/monitoring log += ' | ' log += 'win_rate:[{}], total:[{}], PNL%:[{}]'.format( round(testing_data['win_rate'], 2), testing_data['total'], round(testing_data['pnl_percentage'], 2), ) if testing_data['pnl_percentage'] > 0 and training_data[ 'pnl_percentage'] > 0: log = jh.style(log, 'bold') else: score = 0.0001 # reset store store.reset() return score, log