def generate_parameters(self, parameter_id, **kwargs): """ Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later Parameters ---------- parameter_id : int Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. **kwargs Not used Returns ------- dict One newly generated configuration """ if self.pos == self.population_size - 1: logger.debug('Credit added by one in parameters request') self.credit += 1 self.param_ids.append(parameter_id) raise nni.NoMoreTrialError('No more parameters now.') self.pos += 1 trial_info = self.population[self.pos] trial_info.parameter_id = parameter_id self.running[parameter_id] = trial_info logger.info('Generate parameter : %s', trial_info.hyper_parameters) return split_index(trial_info.hyper_parameters)
def update_search_space(self, search_space): """ Get search space Parameters ---------- search_space : dict Search space """ logger.info('Update search space %s', search_space) self.searchspace_json = search_space self.space = json2space(self.searchspace_json) self.random_state = np.random.RandomState() self.population = [] is_rand = dict() for item in self.space: is_rand[item] = True for i in range(self.population_size): hyper_parameters = json2parameter( self.searchspace_json, is_rand, self.random_state) hyper_parameters = split_index(hyper_parameters) checkpoint_dir = os.path.join(self.all_checkpoint_dir, str(i)) hyper_parameters['load_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) hyper_parameters['save_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) self.population.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=hyper_parameters))
def _generate_individual(self, parameter_id): """ This function will generate the config for a trial. If at the first generation, randomly generates individuals to satisfy self.population_size. Otherwise, random choose a pair of individuals and compare their fitnesses. The worst of the pair will be removed. Copy the best of the pair and mutate it to generate a new individual. Parameters ---------- parameter_id : int Returns ------- dict A group of candidate parameters that evolution tuner generated. """ pos = -1 for i in range(len(self.population)): if self.population[i].result is None: pos = i break if pos != -1: indiv = copy.deepcopy(self.population[pos]) self.population.pop(pos) else: random.shuffle(self.population) # avoid only 1 individual has result if len( self.population ) > 1 and self.population[0].result < self.population[1].result: self.population[0] = self.population[1] # mutation on the worse individual space = json2space(self.searchspace_json, self.population[0].config) is_rand = dict() mutation_pos = space[random.randint(0, len(space) - 1)] for i in range(len(self.space)): is_rand[self.space[i]] = (self.space[i] == mutation_pos) config = json2parameter(self.searchspace_json, is_rand, self.random_state, self.population[0].config) if len(self.population) > 1: self.population.pop(1) indiv = Individual(config=config) # remove "_index" from config and save params-id self.running_trials[parameter_id] = indiv config = split_index(indiv.config) return config
def test_split_index_nested(self): """test for nested search space """ nested_params_with_index = { "layer0": { "_name": "Avg_pool", "pooling_size": { "_index": 1, "_value": 2 } }, "layer1": { "_name": "Empty" }, "layer2": { "_name": "Max_pool", "pooling_size": { "_index": 2, "_value": 3 } }, "layer3": { "_name": "Conv", "kernel_size": { "_index": 3, "_value": 5 }, "output_filters": { "_index": 3, "_value": 64 } } } nested_params = { "layer0": { "_name": "Avg_pool", "pooling_size": 2 }, "layer1": { "_name": "Empty" }, "layer2": { "_name": "Max_pool", "pooling_size": 3 }, "layer3": { "_name": "Conv", "kernel_size": 5, "output_filters": 64 } } params = split_index(nested_params_with_index) self.assertEqual(params, nested_params)
def generate_parameters(self, parameter_id, **kwargs): """ This function will returns a dict of trial (hyper-)parameters, as a serializable object. Parameters ---------- parameter_id : int Returns ------- dict A group of candaidte parameters that evolution tuner generated. """ if not self.population: raise RuntimeError('The population is empty') pos = -1 for i in range(len(self.population)): if self.population[i].result is None: pos = i break if pos != -1: indiv = copy.deepcopy(self.population[pos]) self.population.pop(pos) total_config = indiv.config else: random.shuffle(self.population) if self.population[0].result < self.population[1].result: self.population[0] = self.population[1] # mutation space = json2space(self.searchspace_json, self.population[0].config) is_rand = dict() mutation_pos = space[random.randint(0, len(space) - 1)] for i in range(len(self.space)): is_rand[self.space[i]] = (self.space[i] == mutation_pos) config = json2parameter(self.searchspace_json, is_rand, self.random_state, self.population[0].config) self.population.pop(1) # remove "_index" from config and save params-id total_config = config self.total_data[parameter_id] = total_config config = split_index(total_config) return config
def generate_parameters(self, parameter_id, **kwargs): total_params = self._get_suggestion(random_search=False) # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode if total_params in self.total_data.values(): # but it can cause duplicate parameter rarely total_params = self._get_suggestion(random_search=True) self.total_data[parameter_id] = total_params if self.parallel: self.running_data.append(parameter_id) params = split_index(total_params) return params
def test_split_index_normal(self): """test for normal search space """ normal__params_with_index = { "dropout_rate": { "_index": 1, "_value": 0.9 }, "hidden_size": { "_index": 1, "_value": 512 } } normal__params = {"dropout_rate": 0.9, "hidden_size": 512} params = split_index(normal__params_with_index) self.assertEqual(params, normal__params)
def generate_parameters(self, parameter_id, **kwargs): """ Returns a set of trial (hyper-)parameters, as a serializable object. Parameters ---------- parameter_id : int Returns ------- params : dict """ total_params = self.get_suggestion(random_search=False) # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode if total_params in self.total_data.values(): # but it can cause deplicate parameter rarely total_params = self.get_suggestion(random_search=True) self.total_data[parameter_id] = total_params params = split_index(total_params) return params
def receive_trial_result(self, parameter_id, parameters, value, **kwargs): """ Receive trial's result. if the number of finished trials equals ``self.population_size``, start the next epoch to train the model. Parameters ---------- parameter_id : int Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. parameters : dict Hyper-parameters generated by :meth:`generate_parameters`. value : dict Result from trial (the return value of :func:`nni.report_final_result`). """ logger.info('Get one trial result, id = %d, value = %s', parameter_id, value) value = extract_scalar_reward(value) if self.optimize_mode == OptimizeMode.Minimize: value = -value trial_info = self.running.pop(parameter_id, None) trial_info.score = value self.finished.append(trial_info) self.finished_trials += 1 if self.finished_trials == self.population_size: logger.info('Proceeding to next epoch') self.epoch += 1 self.population = [] self.pos = -1 self.running = {} #exploit and explore self.finished = sorted(self.finished, key=lambda x: x.score, reverse=True) cutoff = int(np.ceil(self.fraction * len(self.finished))) tops = self.finished[:cutoff] bottoms = self.finished[self.finished_trials - cutoff:] for bottom in bottoms: top = np.random.choice(tops) exploit_and_explore(bottom, top, self.factor, self.resample_probability, self.epoch, self.searchspace_json) for trial in self.finished: if trial not in bottoms: trial.clean_id() trial.hyper_parameters[ 'load_checkpoint_dir'] = trial.hyper_parameters[ 'save_checkpoint_dir'] trial.hyper_parameters[ 'save_checkpoint_dir'] = os.path.join( trial.checkpoint_dir, str(self.epoch)) self.finished_trials = 0 for _ in range(self.population_size): trial_info = self.finished.pop() self.population.append(trial_info) while self.credit > 0 and self.pos + 1 < len(self.population): self.credit -= 1 self.pos += 1 parameter_id = self.param_ids.pop() trial_info = self.population[self.pos] trial_info.parameter_id = parameter_id self.running[parameter_id] = trial_info self.send_trial_callback( parameter_id, split_index(trial_info.hyper_parameters))