def scrap_generations(link):
soup = get_body_content(link)
generations = []
generation_divs = soup.find_all(class_="home_models_line gene")
gen = len(generation_divs)
for generation_div in generation_divs:
model_divs = generation_div.find_all("a")
models = scrap_models(model_divs)
num_of_models = len(models)
production_period = generation_div.find("h2").text
production_period = re.findall("\(.*\)", production_period)[0]
production_period = get_year_period(production_period)
generation_image = models[0].model_image
generation_number = gen
generation_obj = Generation(
generation_number,
generation_image,
production_period,
num_of_models,
models,
)
generations.append(generation_obj)
gen -= 1
return generations
def __init__(self,
parent_genotypes: np.ndarray,
possibilites: List[float],
population_of_progeny: int,
maximum_feature=None,
selection: str = "gebv",
max_age: int = 1,
max_number: int = sys.maxsize):
if len(parent_genotypes.shape) != 3 or parent_genotypes.shape[:2] != (
2, 2) or any(_ <= 0 for _ in parent_genotypes.shape):
raise AttributeError(
"Массив генотипов особей задан неверно! Размерность должна быть (2 x 2 x N)"
)
if max_age <= 0:
raise AttributeError(
"Максимальный возраст сущнсоти должен быть >= 1")
if not Selection.selection_implemented(selection):
raise NotImplementedError(f"Селекция {selection} не реализована!")
self.generations: List[Generation] = [
Generation(index=0,
genotypes=list(
Genotype(genotype)
for genotype in parent_genotypes),
population=parent_genotypes.shape[0])
]
self.possibilities = possibilites
self.population_of_progeny = population_of_progeny
self._current_generation = 0
self.maximum_feature = (
self.generations[0].genotypes[0].matrix.shape[1] *
2 if maximum_feature is None else maximum_feature)
self.selection = selection
self.max_age = max_age
self.max_number = max_number
def get_child_generation(self, parent_generation: Generation):
selection = Selection(parent_generation,
possibilites=self.possibilities)
parent1, parent2 = getattr(selection, f"_{self.selection}_selection")()
children = self.get_reproduce(parent1, parent2)
return Generation(index=parent_generation.index + 1,
genotypes=children,
population=len(children))
def filter_generation_for_puberty_period(self, generation):
new_generation = Generation(index=generation.index,
genotypes=list(),
population=0)
for i in generation.genotypes:
if i.age > self.puberty_period:
new_generation.genotypes.append(i)
return new_generation
def get_current_generation():
g = Generation.all()
last_generation = g.order('-saved').get()
if not last_generation:
current_generation = 1
else:
current_generation = last_generation.generation + 1
return current_generation
def _gebv_reduction(self) -> Generation:
sorted_features = sorted(enumerate([
FeatureHelper.gebv_feature(genotype, self.freq)
for genotype in self.generation.genotypes
]),
key=lambda value: value[1],
reverse=True)
indexes = [sorted_features[i][0] for i in range(self.max_population)]
return Generation(
index=self.generation.index,
genotypes=[self.generation.genotypes[i] for i in indexes],
population=self.max_population)
def __init__(self,
parent_genotypes: np.ndarray,
possibilites: List[float],
population_of_progeny: int,
maximum_feature=None,
selection: str = "gebv",
max_age: int = 1,
max_population: int = sys.maxsize,
possibilities_for_selection: List[float] = None,
crosses: int = 1,
puberty_period=0):
if len(parent_genotypes.shape
) != 3 or parent_genotypes.shape[1] != 2 or any(
_ <= 0 for _ in parent_genotypes.shape):
raise AttributeError(
"Массив генотипов особей задан неверно! Размерность должна быть (2 x 2 x N)"
)
if max_age <= 0:
raise AttributeError(
"Максимальный возраст сущности должен быть >= 1")
if not Selection.selection_implemented(selection):
raise NotImplementedError(f"Селекция {selection} не реализована!")
if len(parent_genotypes) // 2 < crosses:
raise AttributeError(
"Число скрещиваний не должно превышать количество родительских особей"
)
if max_age <= puberty_period:
raise AttributeError(
"Внимание! Ваша особь умирает прежде, чем начинает размножаться!"
)
self.generations: List[Generation] = [
Generation(index=0,
genotypes=list(
Genotype(genotype)
for genotype in parent_genotypes),
population=parent_genotypes.shape[0])
]
self.possibilities = possibilites
self.population_of_progeny = population_of_progeny
self._current_generation = 0
self.maximum_feature = (
self.generations[0].genotypes[0].matrix.shape[1] *
2 if maximum_feature is None else maximum_feature)
self.selection = selection
self.max_age = max_age
self.max_population = max_population
self.possibilities_for_selection = possibilities_for_selection
self.crosses = crosses
self.puberty_period = puberty_period
def get_child_generation(self, parent_generation: Generation,
max_generations: int):
if parent_generation.genotypes:
selection = Selection(
parent_generation,
max_generations,
possibilities=self.possibilities_for_selection,
crosses=self.crosses)
parents = getattr(selection, f"_{self.selection}_selection")()
childrens = []
for pair in parents:
childrens += self.get_reproduce(pair[0], pair[1])
#childrens = [self.get_reproduce(pair[0], pair[1]) for pair in parents]
else:
childrens = list()
return Generation(index=parent_generation.index + 1,
genotypes=childrens,
population=len(childrens))
def _ie534_reduction(self, crosses: int = 1) -> Generation:
sorted_features = sorted(enumerate([
FeatureHelper.gebv_feature(genotype)
for genotype in self.generation.genotypes
]),
key=lambda value: value[1],
reverse=True) # sorts by gebv: integer
parents = np.array(self.generation.genotypes)
len_parents = len(parents)
'''
best_part_parents = parents[[value[0] for value in sorted_features[:math.ceil(len(sorted_features) * 0.05)]]]
# choose fix number (len(self.generation.genotypes) * 0.05) of parents with the highest gebv
len_best_part_parents = len(best_part_parents)
'''
parent_matrix = np.zeros((len_parents, len_parents))
for i in range(len_parents):
for j in range(len_parents):
parent_matrix[i, j] = np.sum(
np.maximum(
*np.maximum(parents[i].matrix, parents[j].matrix))
) # the number of locus where at least the one in pair have a desirable allele
# * - unpacking in arguments
sorted_indexes = set()
parent_matrix1 = parent_matrix.copy()
while len(sorted_indexes) < self.max_population:
index_parent1, index_parent2 = np.unravel_index(
parent_matrix1.argmax(), parent_matrix1.shape)
sorted_indexes.add(index_parent2)
sorted_indexes.add(index_parent1)
parent_matrix1[index_parent1, index_parent2] = -1
if len(sorted_indexes) > self.max_population:
sorted_indexes = np.array(sorted_indexes)
v534 = np.array(
sum([parent_matrix[i, j] for j in sorted_indexes if i != j])
for i in sorted_indexes)
np.delete(sorted_indexes, np.argmin(v534))
return Generation(
index=self.generation.index,
genotypes=[self.generation.genotypes[i] for i in sorted_indexes],
population=self.max_population)
def save_stats(stats):
gen = Generation()
gen.generation = stats['generation']
gen.population = stats['population']
gen.mean_fitness = stats['mean fitness']
gen.total_fitness = stats['total fitness']
gen.most_fit_genome = stats['mostfit tags']
gen.most_fit_post_fitness = stats['mostfit fitness']
gen.most_fit_post_url = stats['mostfit url']
gen.least_fit_genome = stats['leastfit tags']
gen.least_fit_post_fitness = stats['leastfit fitness']
gen.least_fit_post_url = stats['leastfit url']
gen.unique_alleles = stats['unique alleles']
gen.n_unique_alleles = stats['n unique alleles']
gen.contributors = stats['contributors']
db.put(gen)
def make_generation(parents: List):
return Generation(
index=0, genotypes=[Genotype(np.array(parent)) for parent in parents])