Пример #1
0
        def handle_thread_exception(args):
            """

            :param args:
            :return:
            """
            if args.exc_type == SystemExit:
                return

            # handle Breaking exceptions
            if args.exc_type in [
                    exceptions.ConfigException, exceptions.RouteNotFound,
                    exceptions.InvalidRoutes,
                    exceptions.CandleNotFoundInDatabase
            ]:
                click.clear()
                print('=' * 30 + ' EXCEPTION TRACEBACK:')
                traceback.print_tb(args.exc_traceback, file=sys.stdout)
                print("=" * 73)
                print(
                    '\n', jh.color('Uncaught Exception:', 'red'),
                    jh.color(
                        '{}: {}'.format(args.exc_type.__name__,
                                        args.exc_value), 'yellow'))
                return

            # send notifications if it's a live session
            if jh.is_live():
                jesse_logger.error('{}: {}'.format(args.exc_type.__name__,
                                                   args.exc_value))

            if jh.is_live() or jh.is_collecting_data():
                logging.error("Uncaught Exception:",
                              exc_info=(args.exc_type, args.exc_value,
                                        args.exc_traceback))
            else:
                print('=' * 30 + ' EXCEPTION TRACEBACK:')
                traceback.print_tb(args.exc_traceback, file=sys.stdout)
                print("=" * 73)
                print(
                    '\n', jh.color('Uncaught Exception:', 'red'),
                    jh.color(
                        '{}: {}'.format(args.exc_type.__name__,
                                        args.exc_value), 'yellow'))

            if jh.is_paper_trading():
                print(
                    jh.color(
                        'An uncaught exception was raised. Check the log file at:\n{}'
                        .format('storage/logs/paper-trade.txt'), 'red'))
            elif jh.is_livetrading():
                print(
                    jh.color(
                        'An uncaught exception was raised. Check the log file at:\n{}'
                        .format('storage/logs/live-trade.txt'), 'red'))
            elif jh.is_collecting_data():
                print(
                    jh.color(
                        'An uncaught exception was raised. Check the log file at:\n{}'
                        .format('storage/logs/collect.txt'), 'red'))
Пример #2
0
    def handle_exception(exc_type, exc_value, exc_traceback) -> None:
        if issubclass(exc_type, KeyboardInterrupt):
            sys.excepthook(exc_type, exc_value, exc_traceback)
            return

        # handle Breaking exceptions
        if exc_type in [
            exceptions.InvalidConfig, exceptions.RouteNotFound, exceptions.InvalidRoutes,
            exceptions.CandleNotFoundInDatabase
        ]:
            click.clear()
            print(f"{'=' * 30} EXCEPTION TRACEBACK:")
            traceback.print_tb(exc_traceback, file=sys.stdout)
            print("=" * 73)
            print(
                '\n',
                jh.color('Uncaught Exception:', 'red'),
                jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')
            )
            return

        # send notifications if it's a live session
        if jh.is_live():
            jesse_logger.error(
                f'{exc_type.__name__}: {exc_value}'
            )

        if jh.is_live() or jh.is_collecting_data():
            logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback))
        else:
            print(f"{'=' * 30} EXCEPTION TRACEBACK:")
            traceback.print_tb(exc_traceback, file=sys.stdout)
            print("=" * 73)
            print(
                '\n',
                jh.color('Uncaught Exception:', 'red'),
                jh.color(f'{exc_type.__name__}: {exc_value}', 'yellow')
            )

        if jh.is_paper_trading():
            print(
                jh.color(
                    'An uncaught exception was raised. Check the log file at:\nstorage/logs/paper-trade.txt',
                    'red'
                )
            )
        elif jh.is_livetrading():
            print(
                jh.color(
                    'An uncaught exception was raised. Check the log file at:\nstorage/logs/live-trade.txt',
                    'red'
                )
            )
        elif jh.is_collecting_data():
            print(
                jh.color(
                    'An uncaught exception was raised. Check the log file at:\nstorage/logs/collect.txt',
                    'red'
                )
            )
Пример #3
0
    def log(self, msg, log_type='info'):
        msg = str(msg)

        if log_type == 'info':
            logger.info(msg)
        elif log_type == 'error':
            logger.error(msg)
        else:
            raise ValueError(f'log_type should be either "info" or "error". You passed {log_type}')
Пример #4
0
 def get_fitness(dna: str, dna_bucket: list) -> None:
     try:
         fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
         dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
     except Exception as e:
         proc = os.getpid()
         logger.error(f'process failed - ID: {str(proc)}')
         logger.error("".join(traceback.TracebackException.from_exception(e).format()))
         raise e
Пример #5
0
def error(msg: str, force_print: bool = False) -> None:
    # send notifications if it's a live session
    if is_live():
        from jesse.services import logger
        logger.error(msg)
        if force_print:
            print(color(msg, 'red'))
    else:
        print(color(msg, 'red'))
Пример #6
0
def terminate_session():
    sync_publish('unexpectedTermination', {
        'message':
        "Session terminated as the result of an uncaught exception",
    })

    jesse_logger.error(
        'Session terminated as the result of an uncaught exception')

    jh.terminate_app()
Пример #7
0
 def get_baby(people: List) -> None:
     try:
         # let's make a baby together LOL
         baby = self.make_love()
         # let's mutate baby's genes, who knows, maybe we create a x-man or something
         baby = self.mutate(baby)
         people.append(baby)
     except Exception as e:
         proc = os.getpid()
         logger.error(f'process failed - ID: {str(proc)}')
         logger.error("".join(traceback.TracebackException.from_exception(e).format()))
         raise e
Пример #8
0
 def get_fitness(dna, dna_bucket):
     try:
         fitness_score, fitness_log = self.fitness(dna)
         dna_bucket.append(
             (dna, fitness_score, fitness_log))
     except Exception as e:
         proc = os.getpid()
         logger.error('process failed - ID: {}'.format(
             str(proc)))
         logger.error("".join(
             traceback.TracebackException.from_exception(
                 e).format()))
         raise e
Пример #9
0
def test_can_log_error_by_firing_event():
    set_up()

    # fire first error event
    logger.error('first error!!!!!')
    first_logged_error = {'time': jh.now(), 'message': 'first error!!!!!'}

    assert store.logs.errors == [first_logged_error]

    # fire second error event
    logger.error('second error!!!!!')
    second_logged_error = {'time': jh.now(), 'message': 'second error!!!!!'}

    assert store.logs.errors == [first_logged_error, second_logged_error]
Пример #10
0
    def log(msg: str, log_type: str = 'info') -> None:
        msg = str(msg)

        if log_type == 'info':
            logger.info(msg)

            if jh.is_live():
                notifier.notify(msg)
        elif log_type == 'error':
            logger.error(msg)

            if jh.is_live():
                notifier.notify(msg)
                notifier.notify_urgently(msg)
        else:
            raise ValueError(f'log_type should be either "info" or "error". You passed {log_type}')
Пример #11
0
 def get_fitness(dna: str, dna_bucket: list) -> None:
     try:
         # check if the DNA is already in the list
         if all(dna_tuple[0] != dna
                for dna_tuple in dna_bucket):
             fitness_score, fitness_log_training, fitness_log_testing = self.fitness(
                 dna)
             dna_bucket.append(
                 (dna, fitness_score, fitness_log_training,
                  fitness_log_testing))
         else:
             raise ValueError(
                 f"Initial Population: Double DNA: {dna}")
     except Exception as e:
         proc = os.getpid()
         logger.error(f'process failed - ID: {str(proc)}')
         logger.error("".join(
             traceback.TracebackException.from_exception(
                 e).format()))
         raise e
Пример #12
0
    def handle_thread_exception(args) -> None:
        if args.exc_type == SystemExit:
            return

        if args.exc_type.__name__ == 'Termination':
            sync_publish('termination', {})
            jh.terminate_app()
        else:
            # send notifications if it's a live session
            if jh.is_live():
                jesse_logger.error(
                    f'{args.exc_type.__name__}: {args.exc_value}')
                jesse_logger.info(str(traceback.format_exc()))

            sync_publish(
                'exception', {
                    'error':
                    f"{args.exc_type.__name__}: {str(args.exc_value)}",
                    'traceback': str(traceback.format_exc())
                })
            terminate_session()
Пример #13
0
    def generate_initial_population(self) -> None:
        """
        generates the initial population
        """
        loop_length = int(self.population_size / self.cpu_cores)

        with click.progressbar(length=loop_length, label='Generating initial population...') as progressbar:
            for i in range(loop_length):
                people = []
                with Manager() as manager:
                    dna_bucket = manager.list([])
                    workers = []

                    def get_fitness(dna: str, dna_bucket: list) -> None:
                        try:
                            fitness_score, fitness_log_training, fitness_log_testing = self.fitness(dna)
                            dna_bucket.append((dna, fitness_score, fitness_log_training, fitness_log_testing))
                        except Exception as e:
                            proc = os.getpid()
                            logger.error(f'process failed - ID: {str(proc)}')
                            logger.error("".join(traceback.TracebackException.from_exception(e).format()))
                            raise e

                    try:
                        for _ in range(self.cpu_cores):
                            dna = ''.join(choices(self.charset, k=self.solution_len))
                            w = Process(target=get_fitness, args=(dna, dna_bucket))
                            w.start()
                            workers.append(w)

                        # join workers
                        for w in workers:
                            w.join()
                            if w.exitcode > 0:
                                logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
                    except KeyboardInterrupt:
                        print(
                            jh.color('Terminating session...', 'red')
                        )

                        # terminate all workers
                        for w in workers:
                            w.terminate()

                        # shutdown the manager process manually since garbage collection cannot won't get to do it for us
                        manager.shutdown()

                        # now we can terminate the main session safely
                        jh.terminate_app()
                    except:
                        raise

                    for d in dna_bucket:
                        people.append({
                            'dna': d[0],
                            'fitness': d[1],
                            'training_log': d[2],
                            'testing_log': d[3]
                        })

                # update dashboard
                click.clear()
                progressbar.update(1)
                print('\n')

                table_items = [
                    ['Started at', jh.timestamp_to_arrow(self.start_time).humanize()],
                    ['Index', f'{len(self.population)}/{self.population_size}'],
                    ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
                    ['Trading Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}'],
                    # TODO: add generated DNAs?
                    # ['-'*10, '-'*10],
                    # ['DNA', people[0]['dna']],
                    # ['fitness', round(people[0]['fitness'], 6)],
                    # ['training|testing logs', people[0]['log']],
                ]
                if jh.is_debugging():
                    table_items.insert(3, ['Population Size', self.population_size])
                    table_items.insert(3, ['Iterations', self.iterations])
                    table_items.insert(3, ['Solution Length', self.solution_len])
                    table_items.insert(3, ['-' * 10, '-' * 10])

                table.key_value(table_items, 'Optimize Mode', alignments=('left', 'right'))

                # errors
                if jh.is_debugging() and len(report.errors()):
                    print('\n')
                    table.key_value(report.errors(), 'Error Logs')

                for p in people:
                    self.population.append(p)

        # sort the population
        self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
Пример #14
0
    def evolve(self) -> List[Any]:
        """
        the main method, that runs the evolutionary algorithm
        """
        # generate the population if starting
        if self.started_index == 0:
            self.generate_initial_population()
            if len(self.population) < 0.5 * self.population_size:
                raise ValueError('Too many errors: less then half of the planned population size could be generated.')

            # if even our best individual is too weak, then we better not continue
            if self.population[0]['fitness'] == 0.0001:
                print(jh.color('Cannot continue because no individual with the minimum fitness-score was found. '
                               'Your strategy seems to be flawed or maybe it requires modifications. ', 'yellow'))
                jh.terminate_app()

        loop_length = int(self.iterations / self.cpu_cores)

        i = self.started_index
        with click.progressbar(length=loop_length, label='Evolving...') as progressbar:
            while i < loop_length:
                with Manager() as manager:
                    people = manager.list([])
                    workers = []

                    def get_baby(people: List) -> None:
                        try:
                            # let's make a baby together LOL
                            baby = self.make_love()
                            # let's mutate baby's genes, who knows, maybe we create a x-man or something
                            baby = self.mutate(baby)
                            people.append(baby)
                        except Exception as e:
                            proc = os.getpid()
                            logger.error(f'process failed - ID: {str(proc)}')
                            logger.error("".join(traceback.TracebackException.from_exception(e).format()))
                            raise e

                    try:
                        for _ in range(self.cpu_cores):
                            w = Process(target=get_baby, args=[people])
                            w.start()
                            workers.append(w)

                        for w in workers:
                            w.join()
                            if w.exitcode > 0:
                                logger.error(f'a process exited with exitcode: {str(w.exitcode)}')
                    except KeyboardInterrupt:
                        print(
                            jh.color('Terminating session...', 'red')
                        )

                        # terminate all workers
                        for w in workers:
                            w.terminate()

                        # shutdown the manager process manually since garbage collection cannot won't get to do it for us
                        manager.shutdown()

                        # now we can terminate the main session safely
                        jh.terminate_app()
                    except:
                        raise

                    # update dashboard
                    click.clear()
                    progressbar.update(1)
                    print('\n')

                    table_items = [
                        ['Started At', jh.timestamp_to_arrow(self.start_time).humanize()],
                        ['Index/Total', f'{(i + 1) * self.cpu_cores}/{self.iterations}'],
                        ['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
                        ['Route', f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}']
                    ]
                    if jh.is_debugging():
                        table_items.insert(
                            3,
                            ['Population Size, Solution Length',
                             f'{self.population_size}, {self.solution_len}']
                        )

                    table.key_value(table_items, 'info', alignments=('left', 'right'))

                    # errors
                    if jh.is_debugging() and len(report.errors()):
                        print('\n')
                        table.key_value(report.errors(), 'Error Logs')

                    print('\n')
                    print('Best DNA candidates:')
                    print('\n')

                    # print fittest individuals
                    if jh.is_debugging():
                        fittest_list = [['Rank', 'DNA', 'Fitness', 'Training log || Testing log'], ]
                    else:
                        fittest_list = [['Rank', 'DNA', 'Training log || Testing log'], ]
                    if self.population_size > 50:
                        number_of_ind_to_show = 15
                    elif self.population_size > 20:
                        number_of_ind_to_show = 10
                    elif self.population_size > 9:
                        number_of_ind_to_show = 9
                    else:
                        raise ValueError('self.population_size cannot be less than 10')

                    for j in range(number_of_ind_to_show):
                        log = f"win-rate: {self.population[j]['training_log']['win-rate']}%, total: {self.population[j]['training_log']['total']}, PNL: {self.population[j]['training_log']['PNL']}% || win-rate: {self.population[j]['testing_log']['win-rate']}%, total: {self.population[j]['testing_log']['total']}, PNL: {self.population[j]['testing_log']['PNL']}%"
                        if self.population[j]['testing_log']['PNL'] is not None and self.population[j]['training_log'][
                            'PNL'] > 0 and self.population[j]['testing_log'][
                            'PNL'] > 0:
                            log = jh.style(log, 'bold')
                        if jh.is_debugging():
                            fittest_list.append(
                                [
                                    j + 1,
                                    self.population[j]['dna'],
                                    self.population[j]['fitness'],
                                    log
                                ],
                            )
                        else:
                            fittest_list.append(
                                [
                                    j + 1,
                                    self.population[j]['dna'],
                                    log
                                ],
                            )

                    if jh.is_debugging():
                        table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'right', 'left'))
                    else:
                        table.multi_value(fittest_list, with_headers=True, alignments=('left', 'left', 'left'))

                    # one person has to die and be replaced with the newborn baby
                    for baby in people:
                        random_index = randint(1, len(self.population) - 1)  # never kill our best perforemr
                        try:
                            self.population[random_index] = baby
                        except IndexError:
                            print('=============')
                            print(f'self.population_size: {self.population_size}')
                            print(f'self.population length: {len(self.population)}')
                            jh.terminate_app()

                        self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))

                        # reaching the fitness goal could also end the process
                        if baby['fitness'] >= self.fitness_goal:
                            progressbar.update(self.iterations - i)
                            print('\n')
                            print(f'fitness goal reached after iteration {i}')
                            return baby

                    # save progress after every n iterations
                    if i != 0 and int(i * self.cpu_cores) % 50 == 0:
                        self.save_progress(i)

                    # store a take_snapshot of the fittest individuals of the population
                    if i != 0 and i % int(100 / self.cpu_cores) == 0:
                        self.take_snapshot(i * self.cpu_cores)

                    i += 1

        print('\n\n')
        print(f'Finished {self.iterations} iterations.')
        return self.population
Пример #15
0
    def evolve(self) -> list:
        """
        the main method, that runs the evolutionary algorithm
        """
        # clear the logs to start from a clean slate
        jh.clear_file('storage/logs/optimize-mode.txt')

        logger.log_optimize_mode('Optimization session started')

        if self.started_index == 0:
            logger.log_optimize_mode(
                f"Generating {self.population_size} population size (random DNAs) using {self.cpu_cores} CPU cores"
            )
            self.generate_initial_population()

            if len(self.population) < 0.5 * self.population_size:
                msg = f'Too many errors! less than half of the expected population size could be generated. Only {len(self.population)} indviduals from planned {self.population_size} are usable.'
                logger.log_optimize_mode(msg)
                raise ValueError(msg)

            # if even our best individual is too weak, then we better not continue
            if self.population[0]['fitness'] == 0.0001:
                msg = 'Cannot continue because no individual with the minimum fitness-score was found. Your strategy seems to be flawed or maybe it requires modifications. '
                logger.log_optimize_mode(msg)
                raise exceptions.InvalidStrategy(msg)

        loop_length = int(self.iterations / self.cpu_cores)

        i = self.started_index
        progressbar = Progressbar(loop_length)
        while i < loop_length:
            with Manager() as manager:
                people_bucket = manager.list([])
                workers = []

                try:
                    for _ in range(self.cpu_cores):
                        mommy = self.select_person()
                        daddy = self.select_person()
                        w = Process(
                            target=create_baby,
                            args=(
                                people_bucket, mommy, daddy, self.solution_len, self.charset,
                                jh.get_config('env.optimization'), router.formatted_routes,
                                router.formatted_extra_routes,
                                self.strategy_hp, self.training_candles, self.testing_candles,
                                self.optimal_total
                            )
                        )
                        w.start()
                        workers.append(w)

                    for w in workers:
                        w.join()
                        if w.exitcode > 0:
                            logger.error(f'a process exited with exitcode: {w.exitcode}')
                except exceptions.Termination:
                    self._handle_termination(manager, workers)

                # update dashboard
                click.clear()
                self.update_progressbar(progressbar)
                # general_info streams
                general_info = {
                    'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
                    'index': f'{(i + 1) * self.cpu_cores}/{self.iterations}',
                    'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
                    'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
                    'average_execution_seconds': self.average_execution_seconds
                }
                if jh.is_debugging():
                    general_info['population_size'] = self.population_size
                    general_info['iterations'] = self.iterations
                    general_info['solution_length'] = self.solution_len
                sync_publish('general_info', general_info)

                if self.population_size > 50:
                    number_of_ind_to_show = 40
                elif self.population_size > 20:
                    number_of_ind_to_show = 15
                elif self.population_size > 9:
                    number_of_ind_to_show = 9
                else:
                    raise ValueError('self.population_size cannot be less than 10')

                best_candidates = [{
                        'rank': j + 1,
                        'dna': self.population[j]['dna'],
                        'fitness': round(self.population[j]['fitness'], 4),
                        'training_win_rate': self.population[j]['training_log']['win-rate'],
                        'training_total_trades': self.population[j]['training_log']['total'],
                        'training_pnl': self.population[j]['training_log']['PNL'],
                        'testing_win_rate': self.population[j]['testing_log']['win-rate'],
                        'testing_total_trades': self.population[j]['testing_log']['total'],
                        'testing_pnl': self.population[j]['testing_log']['PNL'],
                    } for j in range(number_of_ind_to_show)]
                sync_publish('best_candidates', best_candidates)

                # one person has to die and be replaced with the newborn baby
                for baby in people_bucket:
                    # never kill our best performer
                    random_index = randint(1, len(self.population) - 1)
                    self.population[random_index] = baby
                    self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))

                    # reaching the fitness goal could also end the process
                    if baby['fitness'] >= self.fitness_goal:
                        self.update_progressbar(progressbar, finished=True)
                        sync_publish('alert', {
                            'message': f'Fitness goal reached after iteration {i*self.cpu_cores}',
                            'type': 'success'
                        })
                        return baby

                # TODO: bring back progress resumption
                # # save progress after every n iterations
                # if i != 0 and int(i * self.cpu_cores) % 50 == 0:
                #     self.save_progress(i)

                # TODO: bring back
                # # store a take_snapshot of the fittest individuals of the population
                # if i != 0 and i % int(100 / self.cpu_cores) == 0:
                #     self.take_snapshot(i * self.cpu_cores)

                i += 1

        sync_publish('alert', {
            'message': f"Finished {self.iterations} iterations. Check your best DNA candidates, "
                       f"if you don't like any of them, try modifying your strategy.",
            'type': 'success'
        })
        return self.population
Пример #16
0
    def generate_initial_population(self) -> None:
        """
        generates the initial population
        """
        length = int(self.population_size / self.cpu_cores)

        progressbar = Progressbar(length)
        for i in range(length):
            people = []
            with Manager() as manager:
                dna_bucket = manager.list([])
                workers = []

                try:
                    for _ in range(self.cpu_cores):
                        dna = ''.join(choices(self.charset, k=self.solution_len))
                        w = Process(
                            target=get_and_add_fitness_to_the_bucket,
                            args=(
                                dna_bucket, jh.get_config('env.optimization'), router.formatted_routes, router.formatted_extra_routes,
                                self.strategy_hp, dna, self.training_candles, self.testing_candles,
                                self.optimal_total
                            )
                        )
                        w.start()
                        workers.append(w)

                    # join workers
                    for w in workers:
                        w.join()
                        if w.exitcode > 0:
                            logger.error(f'a process exited with exitcode: {w.exitcode}')
                except exceptions.Termination:
                    self._handle_termination(manager, workers)

                for d in dna_bucket:
                    people.append({
                        'dna': d[0],
                        'fitness': d[1],
                        'training_log': d[2],
                        'testing_log': d[3]
                    })

            # update dashboard
            self.update_progressbar(progressbar)

            # general_info streams
            general_info = {
                'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
                'index': f'{(i + 1) * self.cpu_cores}/{self.population_size}',
                'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
                'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
                'average_execution_seconds': self.average_execution_seconds
            }
            if jh.is_debugging():
                general_info['population_size'] = self.population_size
                general_info['iterations'] = self.iterations
                general_info['solution_length'] = self.solution_len
            sync_publish('general_info', general_info)

            for p in people:
                self.population.append(p)

        sync_publish('progressbar', {
            'current': 100,
            'estimated_remaining_seconds': 0
        })
        # sort the population
        self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))
Пример #17
0
    def evolve(self):
        """
        the main method, that runs the evolutionary algorithm
        """
        # generate the population if starting
        if self.started_index == 0:
            self.generate_initial_population()
            if len(self.population) < 0.5 * self.population_size:
                raise ValueError(
                    'Too many errors: less then half of the planned population size could be generated.'
                )

        cores_num = cpu_count()
        loop_length = int(self.iterations / cores_num)

        i = self.started_index
        with click.progressbar(length=loop_length,
                               label='Evolving...') as progressbar:
            while i < loop_length:
                with Manager() as manager:
                    people = manager.list([])
                    workers = []

                    def get_baby(people):
                        try:
                            # let's make a baby together LOL
                            baby = self.make_love()
                            # let's mutate baby's genes, who knows, maybe we create a x-man or something
                            baby = self.mutate(baby)
                            people.append(baby)
                        except Exception as e:
                            proc = os.getpid()
                            logger.error('process failed - ID: {}'.format(
                                str(proc)))
                            logger.error("".join(
                                traceback.TracebackException.from_exception(
                                    e).format()))
                            raise e

                    try:
                        for _ in range(cores_num):
                            w = Process(target=get_baby, args=[people])
                            w.start()
                            workers.append(w)

                        for w in workers:
                            w.join()
                            if w.exitcode > 0:
                                logger.error(
                                    'a process exited with exitcode: {}'.
                                    format(str(w.exitcode)))
                    except KeyboardInterrupt:
                        print(jh.color('Terminating session...', 'red'))

                        # terminate all workers
                        for w in workers:
                            w.terminate()

                        # shutdown the manager process manually since garbage collection cannot won't get to do it for us
                        manager.shutdown()

                        # now we can terminate the main session safely
                        jh.terminate_app()
                    except:
                        raise

                    # update dashboard
                    click.clear()
                    progressbar.update(1)
                    print('\n')

                    table_items = [[
                        'Started At',
                        jh.get_arrow(self.start_time).humanize()
                    ],
                                   [
                                       'Index/Total',
                                       '{}/{}'.format((i + 1) * cores_num,
                                                      self.iterations)
                                   ],
                                   [
                                       'errors/info',
                                       '{}/{}'.format(len(store.logs.errors),
                                                      len(store.logs.info))
                                   ],
                                   [
                                       'Route', '{}, {}, {}, {}'.format(
                                           router.routes[0].exchange,
                                           router.routes[0].symbol,
                                           router.routes[0].timeframe,
                                           router.routes[0].strategy_name)
                                   ]]
                    if jh.is_debugging():
                        table_items.insert(3, [
                            'Population Size, Solution Length',
                            '{}, {}'.format(self.population_size,
                                            self.solution_len)
                        ])

                    table.key_value(table_items,
                                    'info',
                                    alignments=('left', 'right'))

                    # errors
                    if jh.is_debugging() and len(report.errors()):
                        print('\n')
                        table.key_value(report.errors(), 'Error Logs')

                    print('\n')
                    print('Best DNA candidates:')
                    print('\n')

                    # print fittest individuals
                    fittest_list = [
                        [
                            'Rank', 'DNA', 'Fitness',
                            'Training log || Testing log'
                        ],
                    ]
                    if self.population_size > 50:
                        number_of_ind_to_show = 15
                    elif self.population_size > 20:
                        number_of_ind_to_show = 10
                    elif self.population_size > 9:
                        number_of_ind_to_show = 9
                    else:
                        raise ValueError(
                            'self.population_size cannot be less than 10')

                    for j in range(number_of_ind_to_show):
                        fittest_list.append([
                            j + 1, self.population[j]['dna'],
                            self.population[j]['fitness'],
                            self.population[j]['log']
                        ], )
                    table.multi_value(fittest_list,
                                      with_headers=True,
                                      alignments=('left', 'left', 'right',
                                                  'left'))

                    # one person has to die and be replaced with the newborn baby
                    for baby in people:
                        random_index = randint(0, len(self.population) - 1)
                        try:
                            self.population[random_index] = baby
                        except IndexError:
                            print('=============')
                            print('self.population_size: {}'.format(
                                self.population_size))
                            print('self.population length: {}'.format(
                                len(self.population)))
                            jh.terminate_app()

                        self.population = list(
                            sorted(self.population,
                                   key=lambda x: x['fitness'],
                                   reverse=True))

                        # reaching the fitness goal could also end the process
                        if baby['fitness'] >= self.fitness_goal:
                            progressbar.update(self.iterations - i)
                            print('\n')
                            print('fitness goal reached after iteration {}'.
                                  format(i))
                            return baby

                    # save progress after every n iterations
                    if i != 0 and int(i * cores_num) % 50 == 0:
                        self.save_progress(i)

                    # store a take_snapshot of the fittest individuals of the population
                    if i != 0 and i % int(100 / cores_num) == 0:
                        self.take_snapshot(i * cores_num)

                    i += 1

        print('\n\n')
        print('Finished {} iterations.'.format(self.iterations))

        return self.population