def setUp(self):
     self.mock_genome_repository = MagicMock()
     self.mock_cluster_repository = MagicMock()
     self.mock_selection_parameters = {"discarding_by_genome_fitness": 0.2,
                                       "discarding_by_cluster_fitness": 0.2}
     self.genome_selector = GenomeSelector(
         self.mock_genome_repository,
         self.mock_cluster_repository,
         self.mock_selection_parameters
     )
Ejemplo n.º 2
0
    def dynamic_init(self):
        """
        Initializes all parts of NEAT that are dependent upon
        the client.
        """

        self.init_db()

        # Class containing huge configuration object.
        # Loads config from JSON or uses default config.
        self.config = NEATConfig(self._session["config_path"])

        # selecting can mean:
        #   - selecting a single genome for mutation
        #   - selecting two genomes for breeding
        #   - selecting two clusters for combination
        #   - selecting two genomes from two given clusters for inter cluster
        #    breeding (since we don't really want to create ALL combinations)
        self.selector = GenomeSelector(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["selection"]
        )
        # makes decisions lol
        # things like what to do and stuff (breeding or mutation, if clustering
        # is necessary etc)
        self.decision_maker = DecisionMaker(
            self.config.parameters["decision_making"]
        )
        # breeder creates a new genome from two given genomes
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.breeder = Breeder(
            self.config.parameters["breeding"]
        )
        # Mutator creates a new genome from a given genome
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.mutator = Mutator(
            self.gene_repository,
            self.config.parameters["mutating"]
        )
        # Analyst analyzes a given genome and creates an AnalysisResult based on
        # it
        self.analyst = GenomeAnalyst()
        # clusterer divides all existing and active genomes in clusters aka spe-
        # cies
        self.clusterer = GenomeClusterer(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["clustering"]
        )

        self.simulator = Simulator(self.gene_repository)
class test_genomeSelector(TestCase):
    def setUp(self):
        self.mock_genome_repository = MagicMock()
        self.mock_cluster_repository = MagicMock()
        self.mock_selection_parameters = {"discarding_by_genome_fitness": 0.2,
                                          "discarding_by_cluster_fitness": 0.2}
        self.genome_selector = GenomeSelector(
            self.mock_genome_repository,
            self.mock_cluster_repository,
            self.mock_selection_parameters
        )

    def test_getGenomesInCluster(self):
        genomes = []
        genomes_in_tuple = []
        for g in range(1, 10):
            genome = StorageGenome()
            genome.fitness = float(1/g)
            genomes.append(genome)
            genomes_in_tuple.append((genome, genome.fitness))
        self.mock_genome_repository.get_genomes_in_cluster = MagicMock(return_value=genomes)

        self.assertListEqual(genomes_in_tuple, self.genome_selector.get_genomes_in_cluster(0))

    def test_getClusterAreaSortedByFitness(self):
        unsorted = [0.0, 1.0, 2.4, 0.4, 0.9, 0.12, 4.32, 3.2, 5.1, 0.55]
        chosen = [0.4, 0.55, 0.9, 1.0, 2.4, 3.2]
        discard = [0.0, 0.12]
        cluster = []
        cluster_chosen = []
        cluster_discard = []
        for i in unsorted:
            c = Cluster()
            c.fitness = i
            cluster.append(c)
            if i in chosen:
                cluster_chosen.append(c)
            if i in discard:
                cluster_discard.append(c)
        self.mock_cluster_repository.get_current_clusters = MagicMock(return_value=cluster)

        cluster_chosen.sort(key=lambda x: x.fitness)

        self.assertListEqual(cluster_chosen, self.genome_selector.get_cluster_area_sorted_by_fitness(0.2, 0.8))
        self.assertListEqual(cluster_discard, self.genome_selector.select_clusters_for_discarding())

    def test_selectGenomesForBreeding_and_selectGenomesForMutation(self):
        offspring = [10, 10, 10, 10]
        cluster = []
        genome = []
        for i in offspring:
            c = Cluster()
            c.offspring = i
            cluster.append(c)
        for i in range(0, 8):
            g = StorageGenome()
            g.fitness = float(1/(i+1))
            genome.append(g)
        g = StorageGenome()

        self.mock_cluster_repository.get_current_clusters = MagicMock(return_value=cluster)
        self.mock_genome_repository.get_genomes_in_cluster = MagicMock(return_value=genome)
        self.assertEqual(8, len(self.genome_selector.select_genomes_for_breeding(0.2)))
        self.assertEqual(type(g), type(self.genome_selector.select_genomes_for_breeding(0.2)[0][0]))
        self.assertEqual(type(g), type(self.genome_selector.select_genomes_for_mutation(0.2)[0]))
        self.assertEqual(8, len(self.genome_selector.select_genomes_for_mutation(0.2)))

    def test_selectClusterForCombination(self):
        offspring = [10, 10, 10, 10]
        cluster = []
        counter = 0
        for i in offspring:
            c = Cluster()
            c.offspring = i
            c.fitness = float(1/(counter+1))
            cluster.append(c)
            counter += 1
        self.mock_cluster_repository.get_current_clusters = MagicMock(return_value=cluster)
        c = Cluster()
        d = (c, c)
        test_comb = self.genome_selector.select_clusters_for_combination()
        self.assertEqual(type(d), type(self.genome_selector.select_clusters_for_combination()))
        self.assertFalse(
            test_comb[0].__eq__(test_comb[1])
        )

    def test_selectClusterCombination(self):
        offspring = [10, 10]
        cluster = []
        genome = []
        for i in offspring:
            c = Cluster()
            c.offspring = i
            cluster.append(c)
        for i in range(0, 8):
            g = StorageGenome()
            g.fitness = float(1/(i+1))
            genome.append(g)
        self.mock_genome_repository.get_genomes_in_cluster = MagicMock(return_value=genome)
        test_comb = self.genome_selector.select_cluster_combinations(cluster[0], cluster[1], 2)
        g = StorageGenome()
        g_type = (g, g)
        g_list = [g_type]
        self.assertEqual(2, len(test_comb))
        self.assertEqual(type(g_list), type(test_comb))
        self.assertEqual(2, len(test_comb[0]))
        self.assertEqual(type(g_type), type(test_comb[0]))

    def test_selectGenomesForDiscarding(self):
        """
        checks if 20% of worst cluster or genomes were selected
        :return:
        """
        cluster = []
        genome = []
        for i in range(0, 4):
            c = Cluster()
            c.offspring = i
            cluster.append(c)
        for i in range(0, 10):
            g = StorageGenome()
            g.fitness = float(1/(i+1))
            genome.append(g)
        self.mock_cluster_repository.get_current_clusters = MagicMock(return_value=cluster)
        self.mock_genome_repository.get_genomes_in_cluster = MagicMock(return_value=genome)
        self.assertEqual(8, len(self.genome_selector.select_genomes_for_discarding()))
Ejemplo n.º 4
0
class MainDirector(Director):
    def __init__(self, **kwargs):
        """
        :param kwargs:
            - mode:
                - exit: exits the program, default action if nothing is provided.
                - new: creates a new database for a given simulation. requires parame-
                 ter simulation to be set.
                - load: loads a database for a given simulation. requires parameter
                 simulation to be set.
            - simulation: the name of the simulation that should be used. must cor-
              respond to a module name in Simulation.
        :return:
        """
        self._maximum_timeouts = 5000
        self.mode = kwargs.get('mode', 'exit')
        self.selector = None  # type: GenomeSelector
        self.decision_maker = None  # type: DecisionMaker
        self.breeder = None  # type: Breeder
        self.mutator = None  # type: Mutator
        self.analyst = None  # type: GenomeAnalyst
        self.clusterer = None  # type: GenomeClusterer
        self.simulator = None  # type: Simulator
        self.simulation_connector = SimulationConnector()  # type: SimulationConnector
        self.database_connector = None  # type: DatabaseConnector
        self.gene_repository = None  # type: GeneRepository
        self.genome_repository = None  # type: GenomeRepository
        self.cluster_repository = None  # type: ClusterRepository
        self.config = None  # type: NEATConfig
        self._session = None  # type: dict
        self._discarded_genomes_count = 0  # type: int

        if self.mode == 'exit':
            exit()
        elif self.mode == 'run_server':

            startup_check = StartupCheck()
            startup_check.run()

            while True:
                try:
                    self.idle()  # TODO: exit command
                except NetworkProtocolException as e:
                    print(e)
                    pass

    def idle(self):
        """
        Standard method that will be executed if local startup is done.
        In this state, the Director will wait for the client.
        """
        self._session = self.simulation_connector.get_session()

        # Session tokens will identify a client.
        # They can be useful for later parallelization.
        # They also identify the database collections which will be used,
        # so that different users can have their own storage and previous
        # sessions can be loaded from storage.

        self.dynamic_init()  # This can be called after the client has connected

        if debug:
            self.database_connector.clear_collection("genomes")
            self.database_connector.clear_collection("clusters")
            self.database_connector.clear_collection("genes")

        # In case of simulation run:
        self.run()

    def dynamic_init(self):
        """
        Initializes all parts of NEAT that are dependent upon
        the client.
        """

        self.init_db()

        # Class containing huge configuration object.
        # Loads config from JSON or uses default config.
        self.config = NEATConfig(self._session["config_path"])

        # selecting can mean:
        #   - selecting a single genome for mutation
        #   - selecting two genomes for breeding
        #   - selecting two clusters for combination
        #   - selecting two genomes from two given clusters for inter cluster
        #    breeding (since we don't really want to create ALL combinations)
        self.selector = GenomeSelector(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["selection"]
        )
        # makes decisions lol
        # things like what to do and stuff (breeding or mutation, if clustering
        # is necessary etc)
        self.decision_maker = DecisionMaker(
            self.config.parameters["decision_making"]
        )
        # breeder creates a new genome from two given genomes
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.breeder = Breeder(
            self.config.parameters["breeding"]
        )
        # Mutator creates a new genome from a given genome
        # it needs the gene_repository to register new genes and to look up used
        # ones
        self.mutator = Mutator(
            self.gene_repository,
            self.config.parameters["mutating"]
        )
        # Analyst analyzes a given genome and creates an AnalysisResult based on
        # it
        self.analyst = GenomeAnalyst()
        # clusterer divides all existing and active genomes in clusters aka spe-
        # cies
        self.clusterer = GenomeClusterer(
            self.genome_repository,
            self.cluster_repository,
            self.config.parameters["clustering"]
        )

        self.simulator = Simulator(self.gene_repository)

    def init_db(self):

        # database connection is a connection to an arbitrary database that is
        # used to store genes, genomes and nodes
        self.database_connector = DatabaseConnector(
            self._session["session_id"]
        )

        # gene_repository administrates all genes ever created
        self.gene_repository = GeneRepository(
            self.database_connector
        )
        # genome_repository administrates all genomes ever created
        self.genome_repository = GenomeRepository(
            self.database_connector
        )
        # cluster_repository administrates all clusters ever created
        self.cluster_repository = ClusterRepository(
            self.database_connector
        )

    def run(self):
        """
        The main function where the simulation is run, new
        genomes are created and discarded
        This is where the evolutionary magic happens.
        """

        # on new, creates random set of genomes based on configuration inside
        # Simulation.given_simulation.config
        self.decision_maker.reset_time()

        # Init population if its not present yet.
        if len(
                list(self.genome_repository.get_current_population())
        ) < self.config.parameters["clustering"]["max_population"]:
            self.init_population()

        while True:

            # 1. Simulation / wait for client
            timeout_count = 0
            advance_generation = None
            while (timeout_count < self._maximum_timeouts) and \
                    advance_generation is None:
                try:
                    advance_generation = self.perform_simulation_io()
                except NetworkTimeoutException:
                    # TODO: log timeout event
                    timeout_count += 1
            if not timeout_count < self._maximum_timeouts:
                raise NetworkTimeoutException

            # Either:
            #   * go on with loop, generate next generation
            #   * save database for later use, hand out session id to client
            if not advance_generation:
                print("Exiting...")
                exit()  # TODO: archive session / signal worker threads

            # 2. Calculate offspring values

            self.calculate_cluster_offspring()

            # 3. Discarding / Regeneration

            if self.decision_maker.inter_cluster_breeding_time:
                # if it's time to cross-breed, first discard a few clusters
                self.discard_clusters()
                # then combine clusters
                self.crossbreed_clusters()
            else:
                # if it's incest time, first discard a few genomes
                self.discard_genomes()
                # then refill the population
                self.generate_new_genomes()

            # 4. Advance time

            self.decision_maker.advance_time()

    def generate_new_genomes(self):
        """
        Regenerates the population by selecting genomes for
        mutation / breeding, running the generation process and performing analysis.
        :return:
        """

        mutation_percentage = self.decision_maker.mutation_percentage
        genomes_for_mutation = self.selector.select_genomes_for_mutation(mutation_percentage)
        genomes_for_breeding = self.selector.select_genomes_for_breeding(1 - mutation_percentage)
        new_genomes = []

        for genome in genomes_for_mutation:
            new_genome = self.mutator.mutate_genome(genome)
            new_genomes.append(new_genome)

        for genome_one, genome_two in genomes_for_breeding:
            new_genome = self.breeder.breed_genomes(
                genome_one,
                genome_two
            )
            new_genomes.append(new_genome)

        for genome in new_genomes:
            self.analyze_and_insert(genome)

    def crossbreed_clusters(self):
        """
        combines two clusters by breeding genomes of both clusters
        :return:
        """
        cluster_one, cluster_two = self.selector.select_clusters_for_combination()
        for genome_one, genome_two in self.selector.select_cluster_combinations(
                cluster_one,
                cluster_two,
                self._discarded_genomes_count
        ):
            new_genome = self.breeder.breed_genomes(genome_one, genome_two)
            self.analyze_and_insert(new_genome)

        self._discarded_genomes_count = 0

    def analyze_and_insert(self, genome: StorageGenome):

        analysis_genome = AnalysisGenome(self.gene_repository, genome)
        analysis_result = self.analyst.analyze(analysis_genome)
        genome.analysis_result = analysis_result
        self.genome_repository.insert_genome(genome)
        self.clusterer.cluster_genome(genome)

    def calculate_cluster_offspring(self):
        """
        Calculates fitness values and offspring for clusters.
        :return:
        """
        self.clusterer.calculate_cluster_offspring_values()

    def discard_genomes(self):
        """
        discards a number of genomes
        :return:
        """
        for genome in self.selector.select_genomes_for_discarding():
            self.genome_repository.disable_genome(genome.genome_id)

    def discard_clusters(self):
        """
        Discards a number of clusters
        :return:
        """
        for cluster in self.selector.select_clusters_for_discarding():
            genomes_to_discard = self.genome_repository.get_genomes_in_cluster(cluster.cluster_id)
            self._discarded_genomes_count += len(list(genomes_to_discard))
            self.genome_repository.disable_genomes([i.genome_id for i in genomes_to_discard])

    def perform_simulation_io(self):
        genomes = list(self.genome_repository.get_current_population())
        block_count = math.ceil(len(genomes) / self._session["block_size"])
        genome_index = 0
        fitness_values = {}

        for block_id in range(block_count):

            block = genomes[genome_index: genome_index + self._session["block_size"]]
            self.simulation_connector.send_block(block, block_id)
            block_inputs = self.simulation_connector.get_block_inputs(block_id)
            self.simulation_connector.send_block_outputs(
                self.compute_genome_outputs(block_inputs),
                block_id
            )
            fitness_values = {
                **fitness_values,
                **self.simulation_connector.get_fitness_values(block_id)
            }

            genome_index += self._session["block_size"]

        self.update_fitness_values(
            fitness_values
        )
        return self.simulation_connector.get_advance_generation()

    def compute_genome_outputs(
            self,
            block_inputs: Dict[ObjectId, Dict[str, float]]
    ) -> Dict[ObjectId, Dict[str, float]]:
        results = dict({})
        for genome_id, inputs in block_inputs.items():
            storage_genome = self.genome_repository.get_genome_by_id(genome_id)
            outputs = self.simulator.simulate_genome(storage_genome, inputs)
            results[genome_id] = outputs
        return results

    def update_fitness_values(
            self,
            fitness_values: Dict[ObjectId, float]
    ) -> None:
        for genome_id, fitness_value in fitness_values.items():
            self.genome_repository.update_genome_fitness(
                genome_id,
                fitness_value
            )

    def init_population(self):
        population_size = self.config.parameters["clustering"]["max_population"]
        input_labels = self.config.parameters["genomes"]["inputs"]
        output_labels = self.config.parameters["genomes"]["outputs"]
        for i in range(population_size):
            genome = StorageGenome(
                inputs=input_labels,
                outputs=output_labels
            )
            self.analyze_and_insert(genome)