def paper(debug: bool, dev: bool) -> None:
"""
trades in real-time on exchange with PAPER money
"""
validate_cwd()
# set trading mode
from jesse.config import config
config['app']['trading_mode'] = 'papertrade'
register_custom_exception_handler()
# debug flag
config['app']['debug_mode'] = debug
from jesse_live import init
from jesse.services.selectors import get_exchange
live_config = locate('live-config.config')
# validate that the "live-config.py" file exists
if live_config is None:
jh.error(
'You\'re either missing the live-config.py file or haven\'t logged in. Run "jesse login" to fix it.',
True)
jh.terminate_app()
# inject live config
init(config, live_config)
# execute live session
from jesse_live.live_mode import run
run(dev)
def dd(msg: str) -> None:
"""
The dd function dumps the given variables and ends execution of the script.
Used for debugging.
:param msg: str
"""
print(msg)
jh.terminate_app()
def terminate_session():
sync_publish('unexpectedTermination', {
'message':
"Session terminated as the result of an uncaught exception",
})
jesse_logger.error(
'Session terminated as the result of an uncaught exception')
jh.terminate_app()
def initiate_ws(exchange_name: str) -> None:
exchange_class = jh.get_config(
f'app.live_drivers.{exchange_name}')
try:
self.drivers[exchange_name] = exchange_class()
except TypeError:
from jesse_live.info import SUPPORTED_EXCHANGES
exchange_names = ''
for se in SUPPORTED_EXCHANGES:
exchange_names += '\n' + '"' + se['name'] + '"'
error_msg = f'Driver for "{exchange_name}" is not supported yet. Supported exchanges are: {exchange_names}'
jh.error(error_msg, force_print=True)
jh.terminate_app()
def initiate_ws(exchange_name: str) -> None:
from jesse_live.info import SUPPORTED_EXCHANGES, SUPPORTED_EXCHANGES_NAMES
exchange_class = jh.get_config(
f'app.live_drivers.{exchange_name}')
if exchange_name not in SUPPORTED_EXCHANGES_NAMES:
exchange_names = ''
for se in SUPPORTED_EXCHANGES:
exchange_names += f'\n "{se["name"]}"'
error_msg = f'Driver for "{exchange_name}" is not supported yet. Supported exchanges are: {exchange_names}'
jh.error(error_msg, force_print=True)
jh.terminate_app()
self.drivers[exchange_name] = exchange_class()
def run(self):
try:
mp.Process.run(self)
except Exception as e:
if type(e).__name__ == 'Termination':
sync_publish('termination', {})
jh.terminate_app()
else:
sync_publish(
'exception',
{
'error': f'{type(e).__name__}: {e}',
'traceback': str(traceback.format_exc()),
},
)
print('Unhandled exception in the process:')
print(traceback.format_exc())
terminate_session()
def handle_thread_exception(args) -> None:
if args.exc_type == SystemExit:
return
if args.exc_type.__name__ == 'Termination':
sync_publish('termination', {})
jh.terminate_app()
else:
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error(
f'{args.exc_type.__name__}: {args.exc_value}')
jesse_logger.info(str(traceback.format_exc()))
sync_publish(
'exception', {
'error':
f"{args.exc_type.__name__}: {str(args.exc_value)}",
'traceback': str(traceback.format_exc())
})
terminate_session()
def generate_initial_population(self) -> None:
"""
generates the initial population
"""
loop_length = int(self.population_size / self.cpu_cores)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna: str, dna_bucket: list) -> None:
try:
fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'training_log': d[2],
'testing_log': d[3]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index', f'{len(self.population)}/{self.population_size}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-' * 10, '-' * 10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]:
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError('Too many errors: less then half of the planned population size could be generated.')
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
jh.terminate_app()
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people: List) -> None:
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
]
if jh.is_debugging():
table_items.insert(
3,
['Population Size, Solution Length',
f'{self.population_size}, {self.solution_len}']
)
table.key_value(table_items, 'info', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
if jh.is_debugging():
fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
else:
fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
'PNL'] > 0 and self.population[j]['testing_log'][
'PNL'] > 0:
log = jh.style(log, 'bold')
if jh.is_debugging():
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
self.population[j]['fitness'],
log
],
)
else:
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
log
],
)
if jh.is_debugging():
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
else:
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(1, len(self.population) - 1) # never kill our best perforemr
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print(f'self.population_size: {self.population_size}')
print(f'self.population length: {len(self.population)}')
jh.terminate_app()
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print(f'fitness goal reached after iteration {i}')
return baby
# save progress after every n iterations
if i != 0 and int(i * self.cpu_cores) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / self.cpu_cores) == 0:
self.take_snapshot(i * self.cpu_cores)
i += 1
print('\n\n')
print(f'Finished {self.iterations} iterations.')
return self.population
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table.key_value(
[[
'started at',
jh.get_arrow(self.start_time).humanize()
],
[
'index/total', '{}/{}'.format(
(i + 1) * cores_num, self.iterations)
],
[
'population_size, solution_len', '{}, {}'.format(
self.population_size, self.solution_len)
],
[
'route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]],
'info',
alignments=('left', 'right'))
print('\n')
# print fittest individuals
fittest_list = [
['rank', 'DNA', 'fitness', 'training|testing logs'],
]
if self.population_size > 50:
number_of_ind_to_show = 25
elif self.population_size > 20:
number_of_ind_to_show = 20
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population
# create and expose ENV_VALUES
ENV_VALUES = dotenv_values('.env')
if jh.is_unit_testing():
ENV_VALUES['POSTGRES_HOST'] = '127.0.0.1'
ENV_VALUES['POSTGRES_NAME'] = 'jesse_db'
ENV_VALUES['POSTGRES_PORT'] = '5432'
ENV_VALUES['POSTGRES_USERNAME'] = '******'
ENV_VALUES['POSTGRES_PASSWORD'] = '******'
ENV_VALUES['REDIS_HOST'] = 'localhost'
ENV_VALUES['REDIS_PORT'] = '6379'
ENV_VALUES['REDIS_DB'] = 0
ENV_VALUES['REDIS_PASSWORD'] = ''
# validation for existence of .env file
if len(list(ENV_VALUES.keys())) == 0:
jh.error(
'.env file is missing from within your local project. '
'This usually happens when you\'re in the wrong directory. '
'\n\nIf you haven\'t created a Jesse project yet, do that by running: \n'
'jesse make-project {name}\n'
'And then go into that project, and run the same command.',
force_print=True)
os._exit(1)
jh.terminate_app()
# raise FileNotFoundError('.env file is missing from within your local project. This usually happens when you\'re in the wrong directory. You can create one by running "cp .env.example .env"')
if not jh.is_unit_testing() and ENV_VALUES['PASSWORD'] == '':
raise EnvironmentError(
'You forgot to set the PASSWORD in your .env file')
def evolve(self):
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError(
'Too many errors: less then half of the planned population size could be generated.'
)
cores_num = cpu_count()
loop_length = int(self.iterations / cores_num)
i = self.started_index
with click.progressbar(length=loop_length,
label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people):
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error('process failed - ID: {}'.format(
str(proc)))
logger.error("".join(
traceback.TracebackException.from_exception(
e).format()))
raise e
try:
for _ in range(cores_num):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(
'a process exited with exitcode: {}'.
format(str(w.exitcode)))
except KeyboardInterrupt:
print(jh.color('Terminating session...', 'red'))
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [[
'Started At',
jh.get_arrow(self.start_time).humanize()
],
[
'Index/Total',
'{}/{}'.format((i + 1) * cores_num,
self.iterations)
],
[
'errors/info',
'{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
],
[
'Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange,
router.routes[0].symbol,
router.routes[0].timeframe,
router.routes[0].strategy_name)
]]
if jh.is_debugging():
table_items.insert(3, [
'Population Size, Solution Length',
'{}, {}'.format(self.population_size,
self.solution_len)
])
table.key_value(table_items,
'info',
alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
fittest_list = [
[
'Rank', 'DNA', 'Fitness',
'Training log || Testing log'
],
]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError(
'self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
fittest_list.append([
j + 1, self.population[j]['dna'],
self.population[j]['fitness'],
self.population[j]['log']
], )
table.multi_value(fittest_list,
with_headers=True,
alignments=('left', 'left', 'right',
'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(0, len(self.population) - 1)
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print('self.population_size: {}'.format(
self.population_size))
print('self.population length: {}'.format(
len(self.population)))
jh.terminate_app()
self.population = list(
sorted(self.population,
key=lambda x: x['fitness'],
reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print('fitness goal reached after iteration {}'.
format(i))
return baby
# save progress after every n iterations
if i != 0 and int(i * cores_num) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / cores_num) == 0:
self.take_snapshot(i * cores_num)
i += 1
print('\n\n')
print('Finished {} iterations.'.format(self.iterations))
return self.population
def generate_initial_population(self):
"""
generates the initial population
"""
cores_num = cpu_count()
loop_length = int(self.population_size / cores_num)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna, dna_bucket):
fitness_score, fitness_log = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log))
try:
for _ in range(cores_num):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'log': d[2]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.get_arrow(self.start_time).humanize()],
['Index', '{}/{}'.format(len(self.population), self.population_size)],
['Trading Route', '{}, {}, {}, {}'.format(
router.routes[0].exchange, router.routes[0].symbol, router.routes[0].timeframe,
router.routes[0].strategy_name
)],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-'*10, '-'*10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
