def generate(name: str) -> None:
path = '{}'.format(name)
# validate that doesn't create if current directory is inside a Jesse project
ls = os.listdir('.')
is_jesse_project = 'strategies' in ls and 'config.py' in ls and 'storage' in ls and 'routes.py' in ls
if is_jesse_project:
print(
jh.color(
'You are already inside a Jesse project. Check your working directory.'
.format(name), 'red'))
return
# validation for name duplication
exists = os.path.isdir(path)
if exists:
print(jh.color('Project "{}" already exists.'.format(name), 'red'))
return
# generate from ExampleStrategy
dirname, filename = os.path.split(os.path.abspath(__file__))
shutil.copytree('{}/project_template'.format(dirname), path)
# output the location of generated strategy directory
print(
jh.color(
'Your project is created successfully. \nRun "cd {}" to begin algo-trading!'
.format(path, name), 'green'))
# reminder for subscribe and forum
print(
jh.color(
"\nDon't forget to subscribe for future updates at https://Jesse.Trade and make sure to check out the forum for support at https://forum.jesse.trade",
"blue"))
def generate(name: str) -> None:
"""
:param name:
:return:
"""
path = f'strategies/{name}'
# validation for name duplication
exists = os.path.isdir(path)
if exists:
print(jh.color(f'Strategy "{name}" already exists.', 'red'))
return
# generate from ExampleStrategy
dirname, filename = os.path.split(os.path.abspath(__file__))
shutil.copytree(f'{dirname}/ExampleStrategy', path)
with open(f"{path}/__init__.py") as fin:
data = fin.read()
data = data.replace('ExampleStrategy', name)
with open(f"{path}/__init__.py", "wt") as fin:
fin.write(data)
# output the location of generated strategy directory
print(jh.color(f'Strategy created at: {path}', 'green'))
def generate(name):
path = '{}'.format(name)
# validate that doesn't create if current directory is inside a Jesse project
ls = os.listdir('.')
is_jesse_project = 'strategies' in ls and 'config.py' in ls and 'storage' in ls and 'routes.py' in ls
if is_jesse_project:
print(
jh.color(
'You are already inside a Jesse project. Check your working directory.'
.format(name), 'red'))
return
# validation for name duplication
exists = os.path.isdir(path)
if exists:
print(jh.color('Project "{}" already exists.'.format(name), 'red'))
return
# generate from ExampleStrategy
dirname, filename = os.path.split(os.path.abspath(__file__))
shutil.copytree('{}/project_template'.format(dirname), path)
# output the location of generated strategy directory
print(
jh.color(
'Your project is created successfully. \nRun "cd {}" to begin algo-trading!'
.format(path, name), 'green'))
def generate(name: str) -> None:
"""
:param name:
:return:
"""
path = 'strategies/{}'.format(name)
# validation for name duplication
exists = os.path.isdir(path)
if exists:
print(jh.color('Strategy "{}" already exists.'.format(name), 'red'))
return
# generate from ExampleStrategy
dirname, filename = os.path.split(os.path.abspath(__file__))
shutil.copytree('{}/ExampleStrategy'.format(dirname), path)
# replace 'ExampleStrategy' with the name of the new strategy
fin = open("{}/__init__.py".format(path), "rt")
data = fin.read()
data = data.replace('ExampleStrategy', name)
fin.close()
fin = open("{}/__init__.py".format(path), "wt")
fin.write(data)
fin.close()
# output the location of generated strategy directory
print(jh.color('Strategy created at: {}'.format(path), 'green'))
def init():
from pydoc import locate
import os
import sys
import jesse.helpers as jh
# Python version validation.
if jh.python_version() < 3.6:
print(
jh.color(
'Jesse has not beed tested with your Python version ({}), hence it may not work properly. Consider upgrading to >= 3.7'.format(
jh.python_version()),
'red'
)
)
# fix directory issue
sys.path.insert(0, os.getcwd())
ls = os.listdir('.')
is_jesse_project = 'strategies' in ls and 'config.py' in ls and 'storage' in ls and 'routes.py' in ls
if not is_jesse_project:
print(
jh.color(
'Invalid directory. To use Jesse inside notebooks, create notebooks inside the root of a Jesse project.',
'red'
)
)
if is_jesse_project:
local_config = locate('config.config')
from jesse.config import set_config
set_config(local_config)
def candles():
"""
:return:
"""
array = []
candle_keys = []
# add routes
for e in router.routes:
if e.strategy is None:
return
candle_keys.append({
'exchange': e.exchange,
'symbol': e.symbol,
'timeframe': e.timeframe
})
# add extra_routes
for e in router.extra_candles:
candle_keys.append({
'exchange': e[0],
'symbol': e[1],
'timeframe': e[2]
})
# headers
array.append([
'exchange-symbol-timeframe', 'timestamp', 'open', 'close', 'high',
'low'
])
for k in candle_keys:
try:
current_candle = store.candles.get_current_candle(
k['exchange'], k['symbol'], k['timeframe'])
green = is_bullish(current_candle)
bold = k['symbol'] in config['app']['trading_symbols'] and k[
'timeframe'] in config['app']['trading_timeframes']
key = jh.key(k['exchange'], k['symbol'], k['timeframe'])
array.append([
jh.style(key, 'underline' if bold else None),
jh.color(jh.timestamp_to_time(current_candle[0]),
'green' if green else 'red'),
jh.color(str(current_candle[1]), 'green' if green else 'red'),
jh.color(str(current_candle[2]), 'green' if green else 'red'),
jh.color(str(current_candle[3]), 'green' if green else 'red'),
jh.color(str(current_candle[4]), 'green' if green else 'red'),
])
except IndexError:
return
except Exception:
raise
return array
def orders(
) -> List[Union[List[str], List[Union[CharField, str, FloatField]]]]:
array = []
# headers
array.append([
'symbol', 'side', 'type', 'qty', 'price', 'flag', 'status',
'created_at'
])
route_orders = []
for r in router.routes:
r_orders = store.orders.get_orders(r.exchange, r.symbol)
for o in r_orders:
route_orders.append(o)
if not len(route_orders):
return None
route_orders.sort(key=lambda x: x.created_at, reverse=False)
for o in route_orders[::-1][0:5]:
array.append([
o.symbol if o.is_active else jh.color(o.symbol, 'gray'),
jh.color(o.side, 'red') if o.side == 'sell' else jh.color(
o.side, 'green'),
o.type if o.is_active else jh.color(o.type, 'gray'),
o.qty if o.is_active else jh.color(str(o.qty), 'gray'),
o.price if o.is_active else jh.color(str(o.price), 'gray'),
o.flag if o.is_active else jh.color(o.flag, 'gray'),
o.status if o.is_active else jh.color(o.status, 'gray'),
jh.timestamp_to_time(o.created_at)[:19] if o.is_active else
jh.color(jh.timestamp_to_time(o.created_at)[:19], 'gray'),
])
return array
def run(dna=False):
"""
:param dna:
"""
# # # # # # # # # # # # # # # # # # # # # # # #
# trading routes
# # # # # # # # # # # # # # # # # # # # # # # #
arr = []
if not dna:
print(
jh.color('{}{}{}'.format('#' * 25, ' Trading Routes ', '#' * 25),
'blue'))
arr.append(('exchange', 'symbol', 'timeframe', 'strategy name', 'DNA'))
else:
print(jh.color('Translated DNAs into hyper-parameters:', 'blue'))
translated_DNAs_count = 0
for i, r in enumerate(router.routes):
if dna and r.dna:
translated_DNAs_count += 1
StrategyClass = jh.get_strategy_class(r.strategy_name)
hyper_parameters = jh.dna_to_hp(StrategyClass.hyper_parameters(),
r.dna)
table.key_value(hyper_parameters.items(),
r.strategy_name,
uppercase_title=False)
print('\n')
else:
arr.append(
[r.exchange, r.symbol, r.timeframe, r.strategy_name, r.dna])
if not dna:
table.multi_value(arr)
print('\n')
else:
if not translated_DNAs_count:
print('No DNA string found.')
# # # # # # # # # # # # # # # # # # # # # # # #
# extra_candles
# # # # # # # # # # # # # # # # # # # # # # # #
if not dna:
print(
jh.color('{}{}{}'.format('#' * 25, ' Extra Candles ', '#' * 25),
'blue'))
arr = [('exchange', 'symbol', 'timeframe')]
for i, r in enumerate(router.extra_candles):
arr.append([r[0], r[1], r[2]])
table.multi_value(arr)
print('\n')
def run(start_date: str,
finish_date: str,
candles=None,
chart=False,
tradingview=False):
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
print('loading candles...')
candles = _load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
# print candles table
key = '{}-{}'.format(config['app']['trading_exchanges'][0],
config['app']['trading_symbols'][0])
table.key_value(stats.candles(candles[key]['candles']),
'candles',
alignments=('left', 'right'))
print('\n')
# print routes table
table.multi_value(stats.routes(router.routes))
print('\n')
# print guidance for debugging candles
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print(
' Symbol | timestamp | open | close | high | low | volume'
)
# run backtest simulation
simulator(candles)
if not jh.should_execute_silently():
# print trades statistics
if store.completed_trades.count > 0:
print('\n')
table.key_value(report.portfolio_metrics(),
'Metrics',
alignments=('left', 'right'))
print('\n')
# save logs
store_logs(tradingview)
if chart:
charts.portfolio_vs_asset_returns()
else:
print(jh.color('No trades were made.', 'yellow'))
def async_save():
Candle.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'candle: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
candle), 'blue'))
def async_save():
Trade.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'trade: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
trade), 'green'))
def async_save():
Ticker.insert(**d).on_conflict_ignore().execute()
print(
jh.color('ticker: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol, ticker
), 'yellow')
)
def async_save() -> None:
Orderbook.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
f'orderbook: {jh.timestamp_to_time(d["timestamp"])}-{exchange}-{symbol}: [{orderbook[0][0][0]}, {orderbook[0][0][1]}], [{orderbook[1][0][0]}, {orderbook[1][0][1]}]',
'magenta'
)
)
def async_save() -> None:
Candle.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
f"candle: {jh.timestamp_to_time(d['timestamp'])}-{exchange}-{symbol}: {candle}",
'blue'
)
)
def async_save() -> None:
Trade.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
f'trade: {jh.timestamp_to_time(d["timestamp"])}-{exchange}-{symbol}: {trade}',
'green'
)
)
def handle_thread_exception(args):
"""
:param args:
:return:
"""
if args.exc_type == SystemExit:
return
# handle Breaking exceptions
if args.exc_type in [
exceptions.ConfigException, exceptions.RouteNotFound,
exceptions.InvalidRoutes,
exceptions.CandleNotFoundInDatabase
]:
click.clear()
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color(
'{}: {}'.format(args.exc_type.__name__,
args.exc_value), 'yellow'))
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error('{}: {}'.format(args.exc_type.__name__,
args.exc_value))
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:",
exc_info=(args.exc_type, args.exc_value,
args.exc_traceback))
else:
print('=' * 30 + ' EXCEPTION TRACEBACK:')
traceback.print_tb(args.exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n', jh.color('Uncaught Exception:', 'red'),
jh.color(
'{}: {}'.format(args.exc_type.__name__,
args.exc_value), 'yellow'))
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/paper-trade.txt'), 'red'))
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/live-trade.txt'), 'red'))
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\n{}'
.format('storage/logs/collect.txt'), 'red'))
def handle_exception(exc_type, exc_value, exc_traceback) -> None:
if issubclass(exc_type, KeyboardInterrupt):
sys.excepthook(exc_type, exc_value, exc_traceback)
return
# handle Breaking exceptions
if exc_type in [
exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes,
exceptions.CandleNotFoundInDatabase
]:
click.clear()
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n',
jh.color('Uncaught Exception:', 'red'),
jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')
)
return
# send notifications if it's a live session
if jh.is_live():
jesse_logger.error(
f'{exc_type.__name__}: {exc_value}'
)
if jh.is_live() or jh.is_collecting_data():
logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback))
else:
print(f"{'=' * 30} EXCEPTION TRACEBACK:")
traceback.print_tb(exc_traceback, file=sys.stdout)
print("=" * 73)
print(
'\n',
jh.color('Uncaught Exception:', 'red'),
jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')
)
if jh.is_paper_trading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt',
'red'
)
)
elif jh.is_livetrading():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt',
'red'
)
)
elif jh.is_collecting_data():
print(
jh.color(
'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt',
'red'
)
)
def positions():
"""
:return:
"""
array = []
# headers
array.append([
'type', 'strategy', 'symbol', 'opened at', 'qty', 'entry',
'current price', 'PNL (%)'
])
for p in store.positions.storage:
pos = store.positions.storage[p]
if pos.pnl_percentage > 0:
pnl_color = 'green'
elif pos.pnl_percentage < 0:
pnl_color = 'red'
else:
pnl_color = 'black'
if pos.type == 'long':
type_color = 'green'
elif pos.type == 'short':
type_color = 'red'
else:
type_color = 'black'
array.append([
jh.color(pos.type, type_color),
pos.strategy.name,
pos.symbol,
'' if pos.is_close else '{} ago'.format(
jh.readable_duration((jh.now() - pos.opened_at) / 1000, 3)),
pos.qty if abs(pos.qty) > 0 else None,
pos.entry_price,
pos.current_price,
'' if pos.is_close else '{} ({}%)'.format(
jh.color(str(round(pos.pnl, 2)), pnl_color),
jh.color(str(round(pos.pnl_percentage, 4)), pnl_color)),
])
return array
def validate_cwd() -> None:
"""
make sure we're in a Jesse project
"""
if not jh.is_jesse_project():
print(
jh.color(
'Current directory is not a Jesse project. You must run commands from the root of a Jesse project. Read this page for more info: https://docs.jesse.trade/docs/getting-started/#create-a-new-jesse-project',
'red'))
os._exit(1)
def validate_cwd() -> None:
"""
make sure we're in a Jesse project
"""
if not is_jesse_project:
print(
jh.color(
'Current directory is not a Jesse project. You must run commands from the root of a Jesse project.',
'red'))
os._exit(1)
def info(msg):
from jesse.store import store
store.logs.info.append({'time': jh.now(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta'))
if jh.is_live():
msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.info(msg)
def async_save():
Orderbook.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'orderbook: {}-{}-{}: [{}, {}], [{}, {}]'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
# best ask
orderbook[0][0][0], orderbook[0][0][1],
# best bid
orderbook[1][0][0], orderbook[1][0][1]
),
'magenta'
)
)
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red'))
store.logs.errors.append({'time': jh.now(), 'message': msg})
if jh.is_live():
msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.error(msg)
def positions(
) -> List[Union[List[str], List[Union[Union[str, int, None], Any]]]]:
array = [[
'type',
'strategy',
'symbol',
'leverage',
'opened at',
'qty',
'entry',
'current price',
'liq price',
'PNL (%)',
]]
for r in router.routes:
pos = r.strategy.position
if pos.pnl_percentage > 0:
pnl_color = 'green'
elif pos.pnl_percentage < 0:
pnl_color = 'red'
else:
pnl_color = 'black'
if pos.type == 'long':
type_color = 'green'
elif pos.type == 'short':
type_color = 'red'
else:
type_color = 'black'
array.append([
jh.color(pos.type, type_color),
pos.strategy.name,
pos.symbol,
pos.leverage,
'' if pos.is_close else
f'{jh.readable_duration((jh.now_to_timestamp() - pos.opened_at) / 1000, 3)} ago',
pos.qty if abs(pos.qty) > 0 else None,
pos.entry_price,
pos.current_price,
'' if
(pos.liquidation_price is None or np.isnan(pos.liquidation_price)
or pos.liquidation_price == 0) else pos.liquidation_price,
'' if pos.is_close else
f'{jh.color(str(round(pos.pnl, 2)), pnl_color)} ({jh.color(str(round(pos.pnl_percentage, 4)), pnl_color)}%)',
])
return array
def error(msg: str) -> None:
msg = str(msg)
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify_urgently(f"ERROR at \"{jh.get_config('env.identifier')}\" account:\n{msg}")
notify(f'ERROR:\n{msg}')
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color(f'[{jh.timestamp_to_time(jh.now_to_timestamp())}]: {msg}', 'red'))
store.logs.errors.append({'time': jh.now_to_timestamp(), 'message': msg})
if jh.is_live() or jh.is_optimizing():
msg = f"[ERROR | {jh.timestamp_to_time(jh.now_to_timestamp())[:19]}] {msg}"
logging.error(msg)
def init() -> None:
from pydoc import locate
import os
import sys
# fix directory issue
sys.path.insert(0, os.getcwd())
ls = os.listdir('.')
is_jesse_project = 'strategies' in ls and 'config.py' in ls and 'storage' in ls and 'routes.py' in ls
if not is_jesse_project:
print(
jh.color(
'Invalid directory. To use Jesse inside notebooks, create notebooks inside the root of a Jesse project.',
'red'))
if is_jesse_project:
local_config = locate('config.config')
from jesse.config import set_config
set_config(local_config)
def run(start_date: str,
finish_date: str,
candles: Dict[str, Dict[str, Union[str, np.ndarray]]] = None,
chart: bool = False,
tradingview: bool = False,
full_reports: bool = False,
csv: bool = False,
json: bool = False) -> None:
# clear the screen
if not jh.should_execute_silently():
click.clear()
# validate routes
validate_routes(router)
# initiate candle store
store.candles.init_storage(5000)
# load historical candles
if candles is None:
print('loading candles...')
candles = load_candles(start_date, finish_date)
click.clear()
if not jh.should_execute_silently():
# print candles table
key = '{}-{}'.format(config['app']['considering_candles'][0][0],
config['app']['considering_candles'][0][1])
table.key_value(stats.candles(candles[key]['candles']),
'candles',
alignments=('left', 'right'))
print('\n')
# print routes table
table.multi_value(stats.routes(router.routes))
print('\n')
# print guidance for debugging candles
if jh.is_debuggable('trading_candles') or jh.is_debuggable(
'shorter_period_candles'):
print(
' Symbol | timestamp | open | close | high | low | volume'
)
# run backtest simulation
simulator(candles)
if not jh.should_execute_silently():
# print trades metrics
if store.completed_trades.count > 0:
change = []
# calcualte market change
for e in router.routes:
if e.strategy is None:
return
first = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(start_date),
Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
last = Candle.select(Candle.close).where(
Candle.timestamp == jh.date_to_timestamp(finish_date) -
60000, Candle.exchange == e.exchange,
Candle.symbol == e.symbol).first()
change.append(
((last.close - first.close) / first.close) * 100.0)
data = report.portfolio_metrics()
data.append(
['Market Change',
str(round(np.average(change), 2)) + "%"])
print('\n')
table.key_value(data, 'Metrics', alignments=('left', 'right'))
print('\n')
# save logs
store_logs(json, tradingview, csv)
if chart:
charts.portfolio_vs_asset_returns()
# QuantStats' report
if full_reports:
quantstats.quantstats_tearsheet()
else:
print(jh.color('No trades were made.', 'yellow'))
def test_color():
msg_text = 'msg'
msg_color = 'black'
assert jh.color(msg_text, msg_color) == '\x1b[30mmsg\x1b[0m'
def generate_initial_population(self) -> None:
"""
generates the initial population
"""
loop_length = int(self.population_size / self.cpu_cores)
with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
for i in range(loop_length):
people = []
with Manager() as manager:
dna_bucket = manager.list([])
workers = []
def get_fitness(dna: str, dna_bucket: list) -> None:
try:
fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
dna = ''.join(choices(self.charset, k=self.solution_len))
w = Process(target=get_fitness, args=(dna, dna_bucket))
w.start()
workers.append(w)
# join workers
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
for d in dna_bucket:
people.append({
'dna': d[0],
'fitness': d[1],
'training_log': d[2],
'testing_log': d[3]
})
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started at', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index', f'{len(self.population)}/{self.population_size}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'],
# TODO: add generated DNAs?
# ['-'*10, '-'*10],
# ['DNA', people[0]['dna']],
# ['fitness', round(people[0]['fitness'], 6)],
# ['training|testing logs', people[0]['log']],
]
if jh.is_debugging():
table_items.insert(3, ['Population Size', self.population_size])
table_items.insert(3, ['Iterations', self.iterations])
table_items.insert(3, ['Solution Length', self.solution_len])
table_items.insert(3, ['-' * 10, '-' * 10])
table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
for p in people:
self.population.append(p)
# sort the population
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
def evolve(self) -> List[Any]:
"""
the main method, that runs the evolutionary algorithm
"""
# generate the population if starting
if self.started_index == 0:
self.generate_initial_population()
if len(self.population) < 0.5 * self.population_size:
raise ValueError('Too many errors: less then half of the planned population size could be generated.')
# if even our best individual is too weak, then we better not continue
if self.population[0]['fitness'] == 0.0001:
print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
jh.terminate_app()
loop_length = int(self.iterations / self.cpu_cores)
i = self.started_index
with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
while i < loop_length:
with Manager() as manager:
people = manager.list([])
workers = []
def get_baby(people: List) -> None:
try:
# let's make a baby together LOL
baby = self.make_love()
# let's mutate baby's genes, who knows, maybe we create a x-man or something
baby = self.mutate(baby)
people.append(baby)
except Exception as e:
proc = os.getpid()
logger.error(f'process failed - ID: {str(proc)}')
logger.error("".join(traceback.TracebackException.from_exception(e).format()))
raise e
try:
for _ in range(self.cpu_cores):
w = Process(target=get_baby, args=[people])
w.start()
workers.append(w)
for w in workers:
w.join()
if w.exitcode > 0:
logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
except KeyboardInterrupt:
print(
jh.color('Terminating session...', 'red')
)
# terminate all workers
for w in workers:
w.terminate()
# shutdown the manager process manually since garbage collection cannot won't get to do it for us
manager.shutdown()
# now we can terminate the main session safely
jh.terminate_app()
except:
raise
# update dashboard
click.clear()
progressbar.update(1)
print('\n')
table_items = [
['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
]
if jh.is_debugging():
table_items.insert(
3,
['Population Size, Solution Length',
f'{self.population_size}, {self.solution_len}']
)
table.key_value(table_items, 'info', alignments=('left', 'right'))
# errors
if jh.is_debugging() and len(report.errors()):
print('\n')
table.key_value(report.errors(), 'Error Logs')
print('\n')
print('Best DNA candidates:')
print('\n')
# print fittest individuals
if jh.is_debugging():
fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
else:
fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
if self.population_size > 50:
number_of_ind_to_show = 15
elif self.population_size > 20:
number_of_ind_to_show = 10
elif self.population_size > 9:
number_of_ind_to_show = 9
else:
raise ValueError('self.population_size cannot be less than 10')
for j in range(number_of_ind_to_show):
log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
'PNL'] > 0 and self.population[j]['testing_log'][
'PNL'] > 0:
log = jh.style(log, 'bold')
if jh.is_debugging():
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
self.population[j]['fitness'],
log
],
)
else:
fittest_list.append(
[
j + 1,
self.population[j]['dna'],
log
],
)
if jh.is_debugging():
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
else:
table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))
# one person has to die and be replaced with the newborn baby
for baby in people:
random_index = randint(1, len(self.population) - 1) # never kill our best perforemr
try:
self.population[random_index] = baby
except IndexError:
print('=============')
print(f'self.population_size: {self.population_size}')
print(f'self.population length: {len(self.population)}')
jh.terminate_app()
self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
# reaching the fitness goal could also end the process
if baby['fitness'] >= self.fitness_goal:
progressbar.update(self.iterations - i)
print('\n')
print(f'fitness goal reached after iteration {i}')
return baby
# save progress after every n iterations
if i != 0 and int(i * self.cpu_cores) % 50 == 0:
self.save_progress(i)
# store a take_snapshot of the fittest individuals of the population
if i != 0 and i % int(100 / self.cpu_cores) == 0:
self.take_snapshot(i * self.cpu_cores)
i += 1
print('\n\n')
print(f'Finished {self.iterations} iterations.')
return self.population
