def candles():
"""
:return:
"""
array = []
candle_keys = []
# add routes
for e in router.routes:
if e.strategy is None:
return
candle_keys.append({
'exchange': e.exchange,
'symbol': e.symbol,
'timeframe': e.timeframe
})
# add extra_routes
for e in router.extra_candles:
candle_keys.append({
'exchange': e[0],
'symbol': e[1],
'timeframe': e[2]
})
# headers
array.append([
'exchange-symbol-timeframe', 'timestamp', 'open', 'close', 'high',
'low'
])
for k in candle_keys:
try:
current_candle = store.candles.get_current_candle(
k['exchange'], k['symbol'], k['timeframe'])
green = is_bullish(current_candle)
bold = k['symbol'] in config['app']['trading_symbols'] and k[
'timeframe'] in config['app']['trading_timeframes']
key = jh.key(k['exchange'], k['symbol'], k['timeframe'])
array.append([
jh.style(key, 'underline' if bold else None),
jh.color(jh.timestamp_to_time(current_candle[0]),
'green' if green else 'red'),
jh.color(str(current_candle[1]), 'green' if green else 'red'),
jh.color(str(current_candle[2]), 'green' if green else 'red'),
jh.color(str(current_candle[3]), 'green' if green else 'red'),
jh.color(str(current_candle[4]), 'green' if green else 'red'),
])
except IndexError:
return
except Exception:
raise
return array
def test_style():
assert jh.style('test', 'bold') == "\x1b[1mtest\x1b[0m"
assert jh.style('test', 'u') == "\x1b[4mtest\x1b[0m"
def fitness(self, dna) -> tuple:
hp = jh.dna_to_hp(self.strategy_hp, dna)
# init candle store
store.candles.init_storage(5000)
# inject required TRAINING candles to the candle store
for num, c in enumerate(config['app']['considering_candles']):
required_candles.inject_required_candles_to_store(
self.training_initial_candles[num], c[0], c[1])
# run backtest simulation
simulator(self.training_candles, hp)
log = ''
# TODO: some of these have to be dynamic based on how many days it's trading for like for example "total"
# I'm guessing we should accept "optimal" total from command line
if store.completed_trades.count > 5:
training_data = stats.trades(store.completed_trades.trades,
store.app.daily_balance)
total_effect_rate = log10(training_data['total']) / log10(
self.optimal_total)
if total_effect_rate > 1:
total_effect_rate = 1
win_rate = training_data['win_rate']
ratio_config = jh.get_config('env.optimization.ratio', 'sharpe')
if ratio_config == 'sharpe':
ratio = training_data['sharpe_ratio']
elif ratio_config == 'calmar':
ratio = training_data['calmar_ratio']
elif ratio_config == 'sortiono':
ratio = training_data['sortino_ratio']
elif ratio_config == 'omega':
ratio = training_data['omega_ratio']
else:
raise ValueError(
'The entered ratio configuration `{}` for the optimization is unknown. Choose between sharpe, calmar, sortino and omega.'
.format(ratio_config))
if ratio < 0:
score = 0.0001
# reset store
store.reset()
return score, log
ratio_normalized = jh.normalize(ratio, -.5, 4)
# log for debugging/monitoring
log = 'win-rate: {}%, total: {}, PNL: {}%'.format(
int(win_rate * 100),
training_data['total'],
round(training_data['net_profit_percentage'], 2),
)
score = total_effect_rate * ratio_normalized
# perform backtest with testing data. this is using data
# model hasn't trained for. if it works well, there is
# high change it will do good with future data too.
store.reset()
store.candles.init_storage(5000)
# inject required TESTING candles to the candle store
for num, c in enumerate(config['app']['considering_candles']):
required_candles.inject_required_candles_to_store(
self.testing_initial_candles[num], c[0], c[1])
# run backtest simulation
simulator(self.testing_candles, hp)
testing_data = stats.trades(store.completed_trades.trades,
store.app.daily_balance)
# log for debugging/monitoring
log += ' || '
if store.completed_trades.count > 0:
log += 'win-rate: {}%, total: {}, PNL: {}%'.format(
int(testing_data['win_rate'] * 100),
testing_data['total'],
round(testing_data['net_profit_percentage'], 2),
)
if testing_data['net_profit_percentage'] > 0 and training_data[
'net_profit_percentage'] > 0:
log = jh.style(log, 'bold')
else:
log += 'win-rate: -, total: -, PNL%: -'
else:
score = 0.0001
# reset store
store.reset()
return score, log
def evolve(self) -> List[Any]:
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError('Too many errors: less then half of the planned population size could be generated.')
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
jh.terminate_app()
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people: List) -> None:
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
]
if jh.is_debugging():
table_items.insert(
3,
['Population Size, Solution Length',
f'{self.population_size}, {self.solution_len}']
)
table.key_value(table_items, 'info', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
if jh.is_debugging():
fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
else:
fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
'PNL'] > 0 and self.population[j]['testing_log'][
'PNL'] > 0:
log = jh.style(log, 'bold')
if jh.is_debugging():
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
self.population[j]['fitness'],
log
],
)
else:
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
log
],
)
if jh.is_debugging():
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
else:
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(1, len(self.population) - 1) # never kill our best perforemr
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print(f'self.population_size: {self.population_size}')
print(f'self.population length: {len(self.population)}')
jh.terminate_app()
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print(f'fitness goal reached after iteration {i}')
return baby
# save progress after every n iterations
if i != 0 and int(i * self.cpu_cores) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / self.cpu_cores) == 0:
self.take_snapshot(i * self.cpu_cores)
i += 1
print('\n\n')
print(f'Finished {self.iterations} iterations.')
return self.population
def fitness(self, dna) -> tuple:
hp = jh.dna_to_hp(self.strategy_hp, dna)
# init candle store
store.candles.init_storage(5000)
# inject required TRAINING candles to the candle store
required_candles.inject_required_candles_to_store(
self.required_initial_training_candles, self.exchange, self.symbol)
# run backtest simulation
simulator(self.training_candles, hp)
log = ''
# TODO: some of these have to be dynamic based on how many days it's trading for like for example "total"
# I'm guessing we should accept "optimal" total from command line
if store.completed_trades.count > 5:
training_data = stats.trades(store.completed_trades.trades)
optimal_expected_total = 100
total = jh.normalize(training_data['total'], 0, 200)
total_effect_rate = log10(
training_data['total']) / log10(optimal_expected_total)
win_rate = training_data['win_rate']
# log for debugging/monitoring
log = 'win_rate:[{}-{}], total:[{}-{}], PNL%:[{}], TER:[{}]'.format(
round(win_rate, 2), round(training_data['win_rate'], 2),
round(total, 2), training_data['total'],
round(training_data['pnl_percentage'], 2),
round(total_effect_rate, 3))
# the fitness score
score = win_rate * total_effect_rate
# perform backtest with testing data. this is using data
# model hasn't trained for. if it works well, there is
# high change it will do good with future data too.
store.reset()
store.candles.init_storage(5000)
# inject required TESTING candles to the candle store
required_candles.inject_required_candles_to_store(
self.required_initial_testing_candles, self.exchange,
self.symbol)
# run backtest simulation
simulator(self.testing_candles, hp)
testing_data = stats.trades(store.completed_trades.trades)
# log for debugging/monitoring
log += ' | '
log += 'win_rate:[{}], total:[{}], PNL%:[{}]'.format(
round(testing_data['win_rate'], 2),
testing_data['total'],
round(testing_data['pnl_percentage'], 2),
)
if testing_data['pnl_percentage'] > 0 and training_data[
'pnl_percentage'] > 0:
log = jh.style(log, 'bold')
else:
score = 0.0001
# reset store
store.reset()
return score, log
