def set_flops_params_bound(self):
block_choices = [3] * sum(self.graph.stage_repeats)
channel_choices = [7] * sum(self.graph.model.stage_repeats)
max_flops, max_params = get_flop_params(block_choices, channel_choices, self.lookup_table)
block_choices = [0] * sum(self.graph.model.stage_repeats)
channel_choices = [3] * sum(self.graph.model.stage_repeats)
min_flops, min_params = get_flop_params(block_choices, channel_choices, self.lookup_table)
return max_flops, min_flops, max_params, min_params
def evolve(self, population, leader_board, search_iter):
start = time.time()
for i, instance in enumerate(population):
if 'error' not in instance:
if self.logger and i % 50 == 0:
self.logger.info(f"Growing Search [{search_iter:03}] Step [{i:03}] Duration [{(time.time()-start)/60:.2f}]s")
channel_masks = self.graph.get_channel_masks(instance['channel'])
acc = self._get_choice_accuracy(instance['block'], channel_masks)
instance['error'] = 1 - acc
temp = (
deepcopy(instance['error']),
deepcopy(instance['block']),
deepcopy(instance['channel']),
deepcopy(instance['flops']),
deepcopy(instance['param']),
)
leader_board.push(temp)
self.history[f"{acc:.3f}"].append(instance)
population.sort(key=lambda x: x['error'])
parents = population[:self.retain_length]
for instance in population[self.retain_length:]:
if self.random_select > random.random():
parents.append(instance)
# Now find out how many spots we have left to fill.
parents_length = len(parents)
desired_length = len(population) - parents_length
children_to_be_grown = []
# Add children, which are bred from two remaining networks.
while len(children_to_be_grown) < desired_length:
# Get a random mom and dad.
father = random.randint(0, parents_length-1)
mother = random.randint(0, parents_length-1)
# Assuming they aren't the same network...
if father != mother:
father = parents[father]
mother = parents[mother]
# Breed them.
children = self.born(father, mother)
# Add the children one at a time.
for child in children:
# Don't grow larger than desired length.
if len(children_to_be_grown) >= desired_length:
break
flops, param = get_flop_params(child['block'], child['channel'], self.lookup_table)
child['flops'] = flops
child['param'] = param
children_to_be_grown.append(child)
if self.logger:
self.logger.info(f"Evolved Search [{search_iter:03}]")
parents.extend(children_to_be_grown)
self._record_search_history(search_iter)
return parents
def mass_mutation(self, population, size):
alien = []
while len(alien) < size:
instance = population[random.randint(0, len(population)-1)]
instance['block'] = self.crossover_mutate(instance['block'], instance['block'], CHOICE['block'], self.mutate_chance)
instance['channel'] = self.crossover_mutate(instance['channel'], instance['channel'], CHOICE['channel'], self.mutate_chance)
flops, param = get_flop_params(instance['block'], instance['channel'], self.lookup_table)
instance['flops'] = flops
instance['param'] = param
alien.append(instance)
return alien
def mass_crossover(self, population, size):
children = []
while len(children) < size:
father = random.randint(0, len(population)-1)
mother = random.randint(0, len(population)-1)
# Assuming they aren't the same network...
if father != mother:
father = population[father]
mother = population[mother]
child = {}
child['block'] = self.crossover_mutate(father['block'], mother['block'], CHOICE['block'], -1)
child['channel'] = self.crossover_mutate(father['channel'], mother['channel'], CHOICE['channel'], -1)
flops, param = get_flop_params(child['block'], child['channel'], self.lookup_table)
child['flops'] = flops
child['param'] = param
children.append(child)
return children
def build_population(self):
population = []
start = time.time()
while len(population) < self.population_size:
block_choices = self.graph.random_block_choices()
channel_choices = self.graph.random_channel_choices()
flops, param = get_flop_params(block_choices, channel_choices, self.lookup_table)
instance = {}
instance['block'] = block_choices
instance['channel'] = channel_choices
instance['flops'] = flops
instance['param'] = param
population.append(instance)
print("\r Building Population" + f"{int(time.time()-start)}s", end='')
if self.logger:
self.logger.info("Population Built")
return population
def evolve(self, epoch_after_cs, pick_id, find_max_param, max_flops, max_params, min_params, logger=None):
'''
Returns:
selected_child(dict):
a candidate that
'''
g = f"{find_max_param}, {max_flops:.2f}, {max_params:.2f}, {min_params:.2f}"
while g in self.bad_generations:
pick_id, find_max_param, max_flops, max_params, min_params = self.forced_evolution()
g = f"{find_max_param}, {max_flops:.2f}, {max_params:.2f}, {min_params:.2f}"
# Prepare random parents for the initial evolution
while len(self.parents) < self.parent_size:
block_choices = self.graph.random_block_choices()
channel_choices = self.graph.random_channel_choices(epoch_after_cs)
flops, param = get_flop_params(block_choices, channel_choices, self.lookup_table)
candidate = dict()
candidate['block_choices'] = block_choices
candidate['channel_choices'] = channel_choices
candidate['flops'] = flops
candidate['param'] = param
self.parents.append(candidate)
# Breed children
duration = 0.0
while len(self.children) < self.children_size:
start = time.time()
candidate = dict()
# randomly select parents from current pool
mother = random.choice(self.parents)
father = random.choice(self.parents)
# make sure mother and father are different
while father is mother:
mother = random.choice(self.parents)
# breed block choice
block_choices = [0] * len(father['block_choices'])
for i in range(len(block_choices)):
block_choices[i] = random.choice([mother['block_choices'][i], father['block_choices'][i]])
# Mutation: randomly mutate some of the children.
if random.random() < self.mutate_ratio:
block_choices[i] = random.choice(self.source_choice['block_choices'])
# breed channel choice
channel_choices = [0] * len(father['channel_choices'])
for i in range(len(channel_choices)):
channel_choices[i] = random.choice([mother['channel_choices'][i], father['channel_choices'][i]])
# Mutation: randomly mutate some of the children after all channel is warming up.
if random.random() < self.mutate_ratio and epoch_after_cs > 0:
channel_choices[i] = random.choice(self.source_choice['channel_choices'])
flops, param = get_flop_params(block_choices, channel_choices, self.lookup_table)
if epoch_after_cs > 0:
# if flops > max_flop or model_size > upper_params:
if flops < (max_flops-self.flops_interval) or flops > max_flops \
or param < min_params or param > max_params:
duration += time.time() - start
if duration > (self.cfg.SPOS.DURATION - 1): # cost too much time in evolution
if logger:
logger.info("Give up this generation for wasting too much time")
logger.info(g)
self.bad_generations.append(g)
pick_id, find_max_param, max_flops, max_params, min_params = self.forced_evolution()
g = f"{find_max_param}, {max_flops:.2f}, {max_params:.2f}, {min_params:.2f}"
while g in self.bad_generations:
pick_id, find_max_param, max_flops, max_params, min_params = self.forced_evolution()
g = f"{find_max_param}, {max_flops:.2f}, {max_params:.2f}, {min_params:.2f}"
duration = 0.0
start = time.time()
print(f"\r Evolving {int(duration)}s", end = '')
continue
candidate['block_choices'] = block_choices
candidate['channel_choices'] = channel_choices
candidate['flops'] = flops
candidate['param'] = param
self.children.append(candidate)
# Set target and select
self.children.sort(key=lambda cand: cand['param'], reverse=find_max_param)
selected_child = self.children[pick_id]
# Update step for the strolling evolution
self.cur_step += 1
# prepare for next evolve
self.parents = self.children[:self.parent_size]
self.children = []
return selected_child