def evolve(self) -> List[Any]:
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError('Too many errors: less then half of the planned population size could be generated.')
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
jh.terminate_app()
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people: List) -> None:
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
]
if jh.is_debugging():
table_items.insert(
3,
['Population Size, Solution Length',
f'{self.population_size}, {self.solution_len}']
)
table.key_value(table_items, 'info', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
if jh.is_debugging():
fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
else:
fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
'PNL'] > 0 and self.population[j]['testing_log'][
'PNL'] > 0:
log = jh.style(log, 'bold')
if jh.is_debugging():
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
self.population[j]['fitness'],
log
],
)
else:
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
log
],
)
if jh.is_debugging():
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
else:
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(1, len(self.population) - 1) # never kill our best perforemr
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print(f'self.population_size: {self.population_size}')
print(f'self.population length: {len(self.population)}')
jh.terminate_app()
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print(f'fitness goal reached after iteration {i}')
return baby
# save progress after every n iterations
if i != 0 and int(i * self.cpu_cores) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / self.cpu_cores) == 0:
self.take_snapshot(i * self.cpu_cores)
i += 1
print('\n\n')
print(f'Finished {self.iterations} iterations.')
return self.population
def generate_initial_population(self) -> None:
"""
generates the initial population
"""
loop_length = int(self.population_size / self.cpu_cores)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna: str, dna_bucket: list) -> None:
try:
fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'training_log': d[2],
'testing_log': d[3]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index', f'{len(self.population)}/{self.population_size}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-' * 10, '-' * 10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError(
'Too many errors: less then half of the planned population size could be generated.'
)
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error('process failed - ID: {}'.format(
str(proc)))
logger.error("".join(
traceback.TracebackException.from_exception(
e).format()))
raise e
try:
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(
'a process exited with exitcode: {}'.
format(str(w.exitcode)))
except KeyboardInterrupt:
print(jh.color('Terminating session...', 'red'))
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [[
'Started At',
jh.get_arrow(self.start_time).humanize()
],
[
'Index/Total',
'{}/{}'.format((i + 1) * cores_num,
self.iterations)
],
[
'errors/info',
'{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
],
[
'Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]]
if jh.is_debugging():
table_items.insert(3, [
'Population Size, Solution Length',
'{}, {}'.format(self.population_size,
self.solution_len)
])
table.key_value(table_items,
'info',
alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
fittest_list = [
[
'Rank', 'DNA', 'Fitness',
'Training log || Testing log'
],
]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population