class TestGenomeRepository(TestCase):
    def setUp(self):
        self.db = MagicMock()
        self.genome_repository = GenomeRepository(self.db)

    def test_get_new_genome(self):
        self.assertEqual(StorageGenome().__class__, self.genome_repository.get_new_genome().__class__)

    def test_get_current_population(self):
        self.db.find_many = lambda x, y: [{}, {}, {}]
        self.assertListEqual([None, None, None], self.genome_repository.get_current_population())

    def test_get_genome_by_id(self):
        self.db.find_one_by_id = lambda x, y: {}
        self.assertEqual(None, self.genome_repository.get_genome_by_id(ObjectId()))

    def test_get_genomes_in_cluster(self):
        self.db.find_many = lambda x, y: [{}, {}, {}]
        self.assertEqual([None, None, None], self.genome_repository.get_genomes_in_cluster(ObjectId()))

    def test_insert_genome(self):
        self.db.insert_one = lambda x, y: y
        genome = StorageGenome()
        encode = Transformator.Transformator.encode_StorageGenome(genome)
        self.assertDictEqual(encode, self.genome_repository.insert_genome(genome))

    def test_insert_genomes(self):
        self.db.insert_many = lambda x, y: y
        e = []
        g = []
        for i in range(1, 4):
            genome = StorageGenome()
            g.append(genome)
            e.append(Transformator.Transformator.encode_StorageGenome(genome))
        self.assertListEqual(e, self.genome_repository.insert_genomes(g))

    def test_update_genome(self):
        self.db.update_one = lambda x, y, z: False
        self.assertFalse(self.genome_repository.update_genome(StorageGenome()))

    def test_update_genomes(self):
        self.genome_repository.update_genome = lambda x: x
        g = []
        for i in range(1, 4):
            g.append(StorageGenome())
        self.assertListEqual(g, self.genome_repository.update_genomes(g))

    def test_disable_genome(self):
        g = StorageGenome()
        g.is_alive = True

        genome = Transformator.Transformator.encode_StorageGenome(g)
        self.db.find_one_by_id = MagicMock(return_value=genome)
        self.genome_repository.update_genome = lambda x: x

        self.assertFalse(self.genome_repository.disable_genome(g._id).is_alive)

    def test_disable_genomes(self):

        self.db.find_one_by_id = lambda x, y: Transformator.Transformator.encode_StorageGenome(StorageGenome())
        genomes = []
        for i in range(0, 10):
            genomes.append(self.db.find_one_by_id(i, ObjectId()))
        self.genome_repository.update_genomes = lambda x: x

        for i in self.genome_repository.disable_genomes(genomes):
            self.assertFalse(i.is_alive)

    def test_updateGenomeFitness(self):
        genome = StorageGenome()
        genome.fitness = 1.0

        self.genome_repository.get_genome_by_id = lambda x: genome
        self.genome_repository.update_genome = lambda x: x
        self.assertEqual(2.0, self.genome_repository.update_genome_fitness(genome._id, 2.0).fitness)

    def test_updateGenomesFitness(self):
        genomes = []
        for i in range(0, 10):
            g = StorageGenome()
            g.fitness = float(1 / (i + 2))
            genomes.append((g, 4.7))
        self.genome_repository.get_genome_by_id = lambda x: x
        self.genome_repository.update_genomes = lambda x: x
        for i in self.genome_repository.update_genomes_fitness(genomes):
            self.assertEqual(4.7, i.fitness)
Exemplo n.º 2
0
class MainDirector(Director):
    def __init__(self, **kwargs):
        """
        :param kwargs:
            - mode:
                - exit: exits the program, default action if nothing is provided.
                - new: creates a new database for a given simulation. requires parame-
                 ter simulation to be set.
                - load: loads a database for a given simulation. requires parameter
                 simulation to be set.
            - simulation: the name of the simulation that should be used. must cor-
              respond to a module name in Simulation.
        :return:
        """
        self._maximum_timeouts = 5000
        self.mode = kwargs.get('mode', 'exit')
        self.selector = None  # type: GenomeSelector
        self.decision_maker = None  # type: DecisionMaker
        self.breeder = None  # type: Breeder
        self.mutator = None  # type: Mutator
        self.analyst = None  # type: GenomeAnalyst
        self.clusterer = None  # type: GenomeClusterer
        self.simulator = None  # type: Simulator
        self.simulation_connector = SimulationConnector()  # type: SimulationConnector
        self.database_connector = None  # type: DatabaseConnector
        self.gene_repository = None  # type: GeneRepository
        self.genome_repository = None  # type: GenomeRepository
        self.cluster_repository = None  # type: ClusterRepository
        self.config = None  # type: NEATConfig
        self._session = None  # type: dict
        self._discarded_genomes_count = 0  # type: int

        if self.mode == 'exit':
            exit()
        elif self.mode == 'run_server':

            startup_check = StartupCheck()
            startup_check.run()

            while True:
                try:
                    self.idle()  # TODO: exit command
                except NetworkProtocolException as e:
                    print(e)
                    pass

    def idle(self):
        """
        Standard method that will be executed if local startup is done.
        In this state, the Director will wait for the client.
        """
        self._session = self.simulation_connector.get_session()

        # Session tokens will identify a client.
        # They can be useful for later parallelization.
        # They also identify the database collections which will be used,
        # so that different users can have their own storage and previous
        # sessions can be loaded from storage.

        self.dynamic_init()  # This can be called after the client has connected

        if debug:
            self.database_connector.clear_collection("genomes")
            self.database_connector.clear_collection("clusters")
            self.database_connector.clear_collection("genes")

        # In case of simulation run:
        self.run()

    def dynamic_init(self):
        """
        Initializes all parts of NEAT that are dependent upon
        the client.
        """

        self.init_db()

        # Class containing huge configuration object.
        # Loads config from JSON or uses default config.
        self.config = NEATConfig(self._session["config_path"])

        # selecting can mean:
        #   - selecting a single genome for mutation
        #   - selecting two genomes for breeding
        #   - selecting two clusters for combination
        #   - selecting two genomes from two given clusters for inter cluster
        #    breeding (since we don't really want to create ALL combinations)
        self.selector = GenomeSelector(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["selection"]
        )
        # makes decisions lol
        # things like what to do and stuff (breeding or mutation, if clustering
        # is necessary etc)
        self.decision_maker = DecisionMaker(
            self.config.parameters["decision_making"]
        )
        # breeder creates a new genome from two given genomes
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.breeder = Breeder(
            self.config.parameters["breeding"]
        )
        # Mutator creates a new genome from a given genome
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.mutator = Mutator(
            self.gene_repository,
            self.config.parameters["mutating"]
        )
        # Analyst analyzes a given genome and creates an AnalysisResult based on
        # it
        self.analyst = GenomeAnalyst()
        # clusterer divides all existing and active genomes in clusters aka spe-
        # cies
        self.clusterer = GenomeClusterer(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["clustering"]
        )

        self.simulator = Simulator(self.gene_repository)

    def init_db(self):

        # database connection is a connection to an arbitrary database that is
        # used to store genes, genomes and nodes
        self.database_connector = DatabaseConnector(
            self._session["session_id"]
        )

        # gene_repository administrates all genes ever created
        self.gene_repository = GeneRepository(
            self.database_connector
        )
        # genome_repository administrates all genomes ever created
        self.genome_repository = GenomeRepository(
            self.database_connector
        )
        # cluster_repository administrates all clusters ever created
        self.cluster_repository = ClusterRepository(
            self.database_connector
        )

    def run(self):
        """
        The main function where the simulation is run, new
        genomes are created and discarded
        This is where the evolutionary magic happens.
        """

        # on new, creates random set of genomes based on configuration inside
        # Simulation.given_simulation.config
        self.decision_maker.reset_time()

        # Init population if its not present yet.
        if len(
                list(self.genome_repository.get_current_population())
        ) < self.config.parameters["clustering"]["max_population"]:
            self.init_population()

        while True:

            # 1. Simulation / wait for client
            timeout_count = 0
            advance_generation = None
            while (timeout_count < self._maximum_timeouts) and \
                    advance_generation is None:
                try:
                    advance_generation = self.perform_simulation_io()
                except NetworkTimeoutException:
                    # TODO: log timeout event
                    timeout_count += 1
            if not timeout_count < self._maximum_timeouts:
                raise NetworkTimeoutException

            # Either:
            #   * go on with loop, generate next generation
            #   * save database for later use, hand out session id to client
            if not advance_generation:
                print("Exiting...")
                exit()  # TODO: archive session / signal worker threads

            # 2. Calculate offspring values

            self.calculate_cluster_offspring()

            # 3. Discarding / Regeneration

            if self.decision_maker.inter_cluster_breeding_time:
                # if it's time to cross-breed, first discard a few clusters
                self.discard_clusters()
                # then combine clusters
                self.crossbreed_clusters()
            else:
                # if it's incest time, first discard a few genomes
                self.discard_genomes()
                # then refill the population
                self.generate_new_genomes()

            # 4. Advance time

            self.decision_maker.advance_time()

    def generate_new_genomes(self):
        """
        Regenerates the population by selecting genomes for
        mutation / breeding, running the generation process and performing analysis.
        :return:
        """

        mutation_percentage = self.decision_maker.mutation_percentage
        genomes_for_mutation = self.selector.select_genomes_for_mutation(mutation_percentage)
        genomes_for_breeding = self.selector.select_genomes_for_breeding(1 - mutation_percentage)
        new_genomes = []

        for genome in genomes_for_mutation:
            new_genome = self.mutator.mutate_genome(genome)
            new_genomes.append(new_genome)

        for genome_one, genome_two in genomes_for_breeding:
            new_genome = self.breeder.breed_genomes(
                genome_one,
                genome_two
            )
            new_genomes.append(new_genome)

        for genome in new_genomes:
            self.analyze_and_insert(genome)

    def crossbreed_clusters(self):
        """
        combines two clusters by breeding genomes of both clusters
        :return:
        """
        cluster_one, cluster_two = self.selector.select_clusters_for_combination()
        for genome_one, genome_two in self.selector.select_cluster_combinations(
                cluster_one,
                cluster_two,
                self._discarded_genomes_count
        ):
            new_genome = self.breeder.breed_genomes(genome_one, genome_two)
            self.analyze_and_insert(new_genome)

        self._discarded_genomes_count = 0

    def analyze_and_insert(self, genome: StorageGenome):

        analysis_genome = AnalysisGenome(self.gene_repository, genome)
        analysis_result = self.analyst.analyze(analysis_genome)
        genome.analysis_result = analysis_result
        self.genome_repository.insert_genome(genome)
        self.clusterer.cluster_genome(genome)

    def calculate_cluster_offspring(self):
        """
        Calculates fitness values and offspring for clusters.
        :return:
        """
        self.clusterer.calculate_cluster_offspring_values()

    def discard_genomes(self):
        """
        discards a number of genomes
        :return:
        """
        for genome in self.selector.select_genomes_for_discarding():
            self.genome_repository.disable_genome(genome.genome_id)

    def discard_clusters(self):
        """
        Discards a number of clusters
        :return:
        """
        for cluster in self.selector.select_clusters_for_discarding():
            genomes_to_discard = self.genome_repository.get_genomes_in_cluster(cluster.cluster_id)
            self._discarded_genomes_count += len(list(genomes_to_discard))
            self.genome_repository.disable_genomes([i.genome_id for i in genomes_to_discard])

    def perform_simulation_io(self):
        genomes = list(self.genome_repository.get_current_population())
        block_count = math.ceil(len(genomes) / self._session["block_size"])
        genome_index = 0
        fitness_values = {}

        for block_id in range(block_count):

            block = genomes[genome_index: genome_index + self._session["block_size"]]
            self.simulation_connector.send_block(block, block_id)
            block_inputs = self.simulation_connector.get_block_inputs(block_id)
            self.simulation_connector.send_block_outputs(
                self.compute_genome_outputs(block_inputs),
                block_id
            )
            fitness_values = {
                **fitness_values,
                **self.simulation_connector.get_fitness_values(block_id)
            }

            genome_index += self._session["block_size"]

        self.update_fitness_values(
            fitness_values
        )
        return self.simulation_connector.get_advance_generation()

    def compute_genome_outputs(
            self,
            block_inputs: Dict[ObjectId, Dict[str, float]]
    ) -> Dict[ObjectId, Dict[str, float]]:
        results = dict({})
        for genome_id, inputs in block_inputs.items():
            storage_genome = self.genome_repository.get_genome_by_id(genome_id)
            outputs = self.simulator.simulate_genome(storage_genome, inputs)
            results[genome_id] = outputs
        return results

    def update_fitness_values(
            self,
            fitness_values: Dict[ObjectId, float]
    ) -> None:
        for genome_id, fitness_value in fitness_values.items():
            self.genome_repository.update_genome_fitness(
                genome_id,
                fitness_value
            )

    def init_population(self):
        population_size = self.config.parameters["clustering"]["max_population"]
        input_labels = self.config.parameters["genomes"]["inputs"]
        output_labels = self.config.parameters["genomes"]["outputs"]
        for i in range(population_size):
            genome = StorageGenome(
                inputs=input_labels,
                outputs=output_labels
            )
            self.analyze_and_insert(genome)