def _sampling(self, n_samples): pop = Population(n_samples) print(len(pop)) pop_X, pop_hashX = [], [] if self.benchmark_name == 'cifar10' or self.benchmark_name == 'cifar100': allowed_choices = ['I', '1', '2'] i = 0 while i < n_samples: new_X = np.random.choice(allowed_choices, 14) new_hashX = convert_X_to_hashX(new_X) if new_hashX not in pop_hashX: pop[i].set('X', new_X) pop[i].set('hashX', new_hashX) i += 1 else: while len(pop_X) < n_samples: matrix_2D, ops_STRING = create_model() modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) if self.benchmark_api.is_valid(modelspec): hashX = self.benchmark_api.get_module_hash(modelspec) if hashX not in pop_hashX: matrix_1D = decoding_matrix(matrix_2D) ops_INT = decoding_ops(ops_STRING) X = combine_matrix1D_and_opsINT(matrix=matrix_1D, ops=ops_INT) pop_X.append(X) pop_hashX.append(hashX) return pop
def _evaluate(self, x, out, check=False, *args, **kwargs): F = np.full((x.shape[0], self.n_obj), np.nan) if self.problem_name == 'nas101': benchmark_api = kwargs['algorithm'].benchmark_api for i in range(x.shape[0]): cell = api.ModelSpec(matrix=np.array(x[i][:-1], dtype=np.int), ops=x[i][-1].tolist()) module_hash = benchmark_api.get_module_hash(cell) F[i, 0] = (self.benchmark_data[module_hash]['params'] - self.min_max['min_model_params']) / ( self.min_max['max_model_params'] - self.min_max['min_model_params']) F[i, 1] = 1 - self.benchmark_data[module_hash]['val_acc'] self._n_evaluated += 1 elif self.problem_name == 'cifar10' or self.problem_name == 'cifar100': for i in range(x.shape[0]): x_str = ''.join(x[i].tolist()) F[i, 0] = (self.benchmark_data[x_str]['MMACs'] - self.min_max['min_MMACs']) / ( self.min_max['max_MMACs'] - self.min_max['min_MMACs']) F[i, 1] = 1 - self.benchmark_data[x_str]['val_acc'] / 100 self._n_evaluated += 1 out['F'] = F if check: return F
def _sampling(self, n_samples): P = Population(n_samples) P_hashX = [] i = 0 if BENCHMARK_NAME == 'NAS-Bench-101': while i < n_samples: matrix_2D, ops_STRING = create_model() MS = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) if BENCHMARK_API.is_valid(MS): hashX = BENCHMARK_API.get_module_hash(MS) if hashX not in P_hashX: P_hashX.append(hashX) matrix_1D = decoding_matrix(matrix_2D) ops_INT = decoding_ops(ops_STRING) X = combine_matrix1D_and_opsINT(matrix=matrix_1D, ops=ops_INT) F, _ = self.evaluate(X=X, using_surrogate_model=False) P[i].set('X', X) P[i].set('hashX', hashX) P[i].set('F', F) self.update_A(P[i]) i += 1 else: if BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100'\ or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120': l = 6 opt = ['0', '1', '2', '3', '4'] else: l = 14 opt = ['I', '1', '2'] while i < n_samples: X = np.random.choice(opt, l) hashX = convert_to_hashX(X, BENCHMARK_NAME) if hashX not in P_hashX: P_hashX.append(hashX) F, _ = self.evaluate(X=X) P[i].set('X', X) P[i].set('hashX', hashX) P[i].set('F', F) self.update_A(P[i]) i += 1 return P
def encode_(X): hashX = [] for x in X: matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(x) matrix_2D = encoding_matrix(matrix_1D) ops_STRING = encoding_ops(ops_INT) modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) hashX.append(BENCHMARK_API.get_module_hash(modelspec)) return np.array(hashX)
def _do_each_gen(self, first=False): if self.using_surrogate_model: self.alpha = np.mean(self.F_total) if self.m_nEs - self.nEs < 2 * self.m_nEs // 3 or self.n_updates == 15: self.update_model = False if not first: tmp_set_X = np.array(self.tmp_A_X) tmp_set_hashX = np.array(self.tmp_A_hashX) tmp_set = Population(len(self.tmp_A_X)) tmp_set.set('X', tmp_set_X) tmp_set.set('hashX', tmp_set_hashX) for i in range(len(tmp_set)): if (tmp_set[i].get('hashX') not in self.A_hashX) and (tmp_set[i].get('hashX') not in self.DS): F, _ = self.evaluate(tmp_set[i].get('X'), using_surrogate_model=False, count_nE=True) tmp_set[i].set('F', F) self.update_A(tmp_set[i]) if self.n_gen % self.update_model_after_n_gens == 0: data = np.array(self.training_data) self.training_data = [] X = [] Y = [] checked = [] for i in range(len(data)): if BENCHMARK_NAME == 'NAS-Bench-101': matrix_1D, ops_INT = split_to_matrix1D_and_opsINT( data[i].get('X')) matrix_2D = encoding_matrix(matrix_1D) ops_STRING = encoding_ops(ops_INT) modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) hashX = BENCHMARK_API.get_module_hash(modelspec) else: hashX = convert_to_hashX(data[i].get('X'), BENCHMARK_NAME) if (hashX not in checked) and (hashX not in self.DS) and ( hashX not in self.A_hashX): checked.append(hashX) F, _ = self.evaluate(data[i].get('X'), using_surrogate_model=False, count_nE=True) data[i].set('F', F) self.update_A(data[i]) X.append(data[i].get('X')) Y.append(F[1]) for i in range(len(self.A_X)): X.append(self.A_X[i]) Y.append(self.A_F[i][1]) X = np.array(X) Y = np.array(Y) if self.update_model: self.n_updates += 1 if BENCHMARK_NAME == 'NAS-Bench-101': self.surrogate_model.fit(x=X, y=Y) else: self.surrogate_model.fit(x=encode(X, BENCHMARK_NAME), y=Y, verbose=False) if DEBUG: print(f'Number of evaluations used: {self.nEs}/{self.m_nEs}') if SAVE: pf = np.array(self.A_F) pf = np.unique(pf, axis=0) pf = pf[np.argsort(pf[:, 0])] pk.dump( [pf, self.nEs], open( f'{self.path}/pf_eval/pf_and_evaluated_gen_{self.n_gen}.p', 'wb')) self.worst_f0 = max(self.worst_f0, np.max(pf[:, 0])) self.worst_f1 = max(self.worst_f1, np.max(pf[:, 1])) dpfs = round(cal_dpfs(pareto_s=pf, pareto_front=BENCHMARK_PF_TRUE), 6) print(self.nEs, dpfs) if len(self.no_eval) == 0: self.dpfs.append(dpfs) self.no_eval.append(self.nEs) else: if self.nEs == self.no_eval[-1]: self.dpfs[-1] = dpfs else: self.dpfs.append(dpfs) self.no_eval.append(self.nEs)
def local_search_on_X(self, P, X, ls_on_knee_solutions=False): P_hashX = P.get('hashX') len_soln = len(P_hashX[0]) S = P.new() S = S.merge(X) # Using for local search on knee solutions first, last = 0, 0 if ls_on_knee_solutions: first, last = len(S) - 2, len(S) - 1 m_searches = len_soln if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100': ops = ['I', '1', '2'] else: ops = ['0', '1', '2', '3', '4'] for i in range(len(S)): # Avoid stuck because don't find any better architecture tmp_m_searches = 100 tmp_n_searches = 0 checked = [S[i].get('hashX')] n_searches = 0 while (n_searches < m_searches) and (tmp_n_searches < tmp_m_searches): tmp_n_searches += 1 o = S[i].copy() # --> neighboring solution if BENCHMARK_NAME == 'NAS-Bench-101': ''' Local search on edges ''' matrix_1D, ops_INT = split_to_matrix1D_and_opsINT( o.get('X')) ops_STRING = encoding_ops(ops_INT) while True: idxs = np.random.choice(range(len(matrix_1D)), size=self.n_vars, replace=False) matrix_1D_new = matrix_1D.copy() matrix_1D_new[idxs] = 1 - matrix_1D[idxs] matrix_2D_new = encoding_matrix(matrix_1D_new) modelspec = api.ModelSpec(matrix=matrix_2D_new, ops=ops_STRING) if BENCHMARK_API.is_valid(modelspec): o_hashX = BENCHMARK_API.get_module_hash(modelspec) o_X = combine_matrix1D_and_opsINT( matrix_1D_new, ops_INT) break else: idxs = np.random.choice(range(len(o.get('X'))), size=self.n_vars, replace=False) o_X = o.get('X').copy() for idx in idxs: allowed_ops = ops.copy() allowed_ops.remove(o.get('X')[idx]) new_op = np.random.choice(allowed_ops) o_X[idx] = new_op o_hashX = convert_to_hashX(o_X, BENCHMARK_NAME) if (o_hashX not in checked) and (o_hashX not in P_hashX) and ( o_hashX not in self.DS): checked.append(o_hashX) n_searches += 1 o_F, twice = self.evaluate( o_X, using_surrogate_model=self.using_surrogate_model) if i == first and LOCAL_SEARCH_ON_KNEE_SOLUTIONS: better_idv = find_better_idv(o_F, S[i].get('F'), 'first') elif i == last and LOCAL_SEARCH_ON_KNEE_SOLUTIONS: better_idv = find_better_idv(o_F, S[i].get('F'), 'last') else: better_idv = find_better_idv(o_F, S[i].get('F')) if better_idv == 1: # --> neighboring solutions is better S[i].set('X', o_X) S[i].set('hashX', o_hashX) S[i].set('F', o_F) if not self.using_surrogate_model: self.update_A(S[i]) else: if twice: self.update_A(S[i]) else: self.training_data.append(S[i]) self.update_fake_A(S[i]) else: # --> no one is better || current solution is better o.set('X', o_X) o.set('hashX', o_hashX) o.set('F', o_F) if not self.using_surrogate_model: self.update_A(o) else: if twice: self.update_A(o) else: self.training_data.append(o) self.update_fake_A(o) return S
def _mutation(self, P, O): P_hashX = P.get('hashX') new_O = Population(len(O)) new_O_hashX = [] old_O_X = O.get('X') i = 0 full = False pM = 1 / len(old_O_X[0]) while not full: if BENCHMARK_NAME == 'NAS-Bench-101': for x in old_O_X: matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(x) new_matrix_1D = matrix_1D.copy() new_ops_INT = ops_INT.copy() pM_idxs_matrix = np.random.rand(len(new_matrix_1D)) for j in range(len(pM_idxs_matrix)): if pM_idxs_matrix[j] <= pM: new_matrix_1D[j] = 1 - new_matrix_1D[j] pM_idxs_ops = np.random.rand(len(new_ops_INT)) for j in range(len(pM_idxs_ops)): if pM_idxs_ops[j] <= pM: choices = [0, 1, 2] choices.remove(new_ops_INT[j]) new_ops_INT[j] = np.random.choice(choices) matrix_2D = encoding_matrix(new_matrix_1D) ops_STRING = encoding_ops(new_ops_INT) new_MS = api.ModelSpec(matrix_2D, ops_STRING) if BENCHMARK_API.is_valid(new_MS): hashX = BENCHMARK_API.get_module_hash(new_MS) if (hashX not in new_O_hashX) and ( hashX not in P_hashX) and (hashX not in self.DS): X = combine_matrix1D_and_opsINT( new_matrix_1D, new_ops_INT) new_O_hashX.append(hashX) F, twice = self.evaluate( X=X, using_surrogate_model=self. using_surrogate_model) new_O[i].set('X', X) new_O[i].set('hashX', hashX) new_O[i].set('F', F) if not self.using_surrogate_model: self.update_A(new_O[i]) else: if twice: self.update_A(new_O[i]) else: self.training_data.append(new_O[i]) self.update_fake_A(new_O[i]) i += 1 if i == len(P): full = True break else: if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100': opt = ['I', '1', '2'] else: opt = ['0', '1', '2', '3', '4'] pM_idxs = np.random.rand(old_O_X.shape[0], old_O_X.shape[1]) for m in range(len(old_O_X)): X = old_O_X[m].copy() for n in range(pM_idxs.shape[1]): if pM_idxs[m][n] <= pM: allowed_opt = opt.copy() allowed_opt.remove(X[n]) X[n] = np.random.choice(allowed_opt) hashX = convert_to_hashX(X, BENCHMARK_NAME) if (hashX not in new_O_hashX) and ( hashX not in P_hashX) and (hashX not in self.DS): new_O_hashX.append(hashX) F, twice = self.evaluate( X=X, using_surrogate_model=self.using_surrogate_model) new_O[i].set('X', X) new_O[i].set('hashX', hashX) new_O[i].set('F', F) if not self.using_surrogate_model: self.update_A(new_O[i]) else: if twice: self.update_A(new_O[i]) else: self.training_data.append(new_O[i]) self.update_fake_A(new_O[i]) i += 1 if i == len(P): full = True break return new_O
def _crossover(self, P, pC=0.9): O = Population(len(P)) O_hashX = [] nCOs = 0 # --> Avoid to stuck i = 0 full = False while not full: idx = np.random.choice(len(P), size=(len(P) // 2, 2), replace=False) P_ = P[idx] if BENCHMARK_NAME == 'NAS-Bench-101': for j in range(len(P_)): if np.random.random() < pC: new_O1_X, new_O2_X = crossover(P_[j][0].get('X'), P_[j][1].get('X'), self.typeC) matrix1_1D, ops1_INT = split_to_matrix1D_and_opsINT( new_O1_X) matrix2_1D, ops2_INT = split_to_matrix1D_and_opsINT( new_O2_X) matrix1_2D = encoding_matrix(matrix1_1D) matrix2_2D = encoding_matrix(matrix2_1D) ops1_STRING = encoding_ops(ops1_INT) ops2_STRING = encoding_ops(ops2_INT) new_MS1 = api.ModelSpec(matrix=matrix1_2D, ops=ops1_STRING) new_MS2 = api.ModelSpec(matrix=matrix2_2D, ops=ops2_STRING) new_MS_lst = [new_MS1, new_MS2] new_O_X_lst = [new_O1_X, new_O2_X] for m in range(2): if BENCHMARK_API.is_valid(new_MS_lst[m]): new_O_hashX = BENCHMARK_API.get_module_hash( new_MS_lst[m]) if nCOs <= 100: if (new_O_hashX not in O_hashX) and ( new_O_hashX not in self.DS): O_hashX.append(new_O_hashX) new_O_F, twice = self.evaluate( X=new_O_X_lst[m], using_surrogate_model=self. using_surrogate_model) O[i].set('X', new_O_X_lst[m]) O[i].set('hashX', new_O_hashX) O[i].set('F', new_O_F) if not self.using_surrogate_model: self.update_A(O[i]) else: if twice: self.update_A(O[i]) else: self.training_data.append(O[i]) self.update_fake_A(O[i]) i += 1 if i == len(P): full = True break else: O_hashX.append(new_O_hashX) new_O_F, twice = self.evaluate( X=new_O_X_lst[m], using_surrogate_model=self. using_surrogate_model) O[i].set('X', new_O_X_lst[m]) O[i].set('hashX', new_O_hashX) O[i].set('F', new_O_F) if not self.using_surrogate_model: self.update_A(O[i]) else: if twice: self.update_A(O[i]) else: self.training_data.append(O[i]) self.update_fake_A(O[i]) i += 1 if i == len(P): full = True break else: for m in range(2): O[i].set('X', P_[j][m].get('X')) O[i].set('hashX', P_[j][m].get('hashX')) O[i].set('F', P_[j][m].get('F')) i += 1 if i == len(P): full = True break if full: break else: for j in range(len(P_)): if np.random.random() < pC: o1_X, o2_X = crossover(P_[j][0].get('X'), P_[j][1].get('X'), self.typeC) o_X = [o1_X, o2_X] o_hashX = [ convert_to_hashX(o1_X, BENCHMARK_NAME), convert_to_hashX(o2_X, BENCHMARK_NAME) ] if nCOs <= 100: for m in range(2): if (o_hashX[m] not in O_hashX) and (o_hashX[m] not in self.DS): O_hashX.append(o_hashX[m]) o_F, twice = self.evaluate( X=o_X[m], using_surrogate_model=self. using_surrogate_model) O[i].set('X', o_X[m]) O[i].set('hashX', o_hashX[m]) O[i].set('F', o_F) if not self.using_surrogate_model: self.update_A(O[i]) else: if twice: self.update_A(O[i]) else: self.training_data.append(O[i]) self.update_fake_A(O[i]) i += 1 if i == len(P): full = True break else: for m in range(2): O_hashX.append(o_hashX[m]) o_F, twice = self.evaluate( X=o_X[m], using_surrogate_model=self. using_surrogate_model) O[i].set('X', o_X[m]) O[i].set('hashX', o_hashX[m]) O[i].set('F', o_F) if not self.using_surrogate_model: self.update_A(O[i]) else: if twice: self.update_A(O[i]) else: self.training_data.append(O[i]) self.update_fake_A(O[i]) i += 1 if i == len(P): full = True break else: for m in range(2): O[i].set('X', P_[j][m].get('X')) O[i].set('hashX', P_[j][m].get('hashX')) O[i].set('F', P_[j][m].get('F')) i += 1 if i == len(P): full = True break if full: break nCOs += 1 return O
def evaluate(self, X, using_surrogate_model=False, count_nE=True): F = np.full(2, fill_value=np.nan) twice = False # --> using on 'surrogate model method' if not using_surrogate_model: if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100': hashX = ''.join(X.tolist()) F[0] = np.round( (BENCHMARK_DATA[hashX]['MMACs'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'] / 100, 6) elif BENCHMARK_NAME == 'NAS-Bench-101': matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(X) matrix_2D = encoding_matrix(matrix_1D) ops_STRING = encoding_ops(ops_INT) modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) hashX = BENCHMARK_API.get_module_hash(modelspec) F[0] = np.round( (BENCHMARK_DATA[hashX]['params'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'], 6) elif BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100' \ or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120': hashX = convert_to_hashX(X, BENCHMARK_NAME) F[0] = np.round( (BENCHMARK_DATA[hashX]['FLOP'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = np.round( 1 - BENCHMARK_DATA[hashX]['test-accuracy'] / 100, 6) if count_nE: self.nEs += 1 self.F_total.append(F[1]) else: if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100': encode_X = encode(X, BENCHMARK_NAME) hashX = ''.join(X.tolist()) F[0] = np.round( (BENCHMARK_DATA[hashX]['MMACs'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = self.surrogate_model.predict(np.array([encode_X]))[0][0] if F[1] < self.alpha: twice = True F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'] / 100, 6) self.nEs += 1 elif BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100' \ or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120': hashX = convert_to_hashX(X, BENCHMARK_NAME) encode_X = encode(X, BENCHMARK_NAME) F[0] = np.round( (BENCHMARK_DATA[hashX]['FLOP'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = self.surrogate_model.predict(np.array([encode_X]))[0][0] if F[1] < self.alpha: twice = True F[1] = np.round( 1 - BENCHMARK_DATA[hashX]['test-accuracy'] / 100, 6) self.nEs += 1 else: matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(X) matrix_2D = encoding_matrix(matrix_1D) ops_STRING = encoding_ops(ops_INT) modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING) hashX = BENCHMARK_API.get_module_hash(modelspec) F[0] = np.round( (BENCHMARK_DATA[hashX]['params'] - BENCHMARK_MIN_MAX[0]) / (BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6) F[1] = self.surrogate_model.predict(np.array([X]))[0][0] if F[1] < self.alpha: twice = True F[1] = 1 - BENCHMARK_DATA[hashX]['val_acc'] self.nEs += 1 if twice: self.F_total.append(F[1]) return F, twice
def get_model_spec(X): edges_matrix, ops_matrix = X2matrices(X) return api.ModelSpec(edges_matrix, ops_matrix)