def _sampling(self, n_samples):
pop = Population(n_samples)
print(len(pop))
pop_X, pop_hashX = [], []
if self.benchmark_name == 'cifar10' or self.benchmark_name == 'cifar100':
allowed_choices = ['I', '1', '2']
i = 0
while i < n_samples:
new_X = np.random.choice(allowed_choices, 14)
new_hashX = convert_X_to_hashX(new_X)
if new_hashX not in pop_hashX:
pop[i].set('X', new_X)
pop[i].set('hashX', new_hashX)
i += 1
else:
while len(pop_X) < n_samples:
matrix_2D, ops_STRING = create_model()
modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING)
if self.benchmark_api.is_valid(modelspec):
hashX = self.benchmark_api.get_module_hash(modelspec)
if hashX not in pop_hashX:
matrix_1D = decoding_matrix(matrix_2D)
ops_INT = decoding_ops(ops_STRING)
X = combine_matrix1D_and_opsINT(matrix=matrix_1D, ops=ops_INT)
pop_X.append(X)
pop_hashX.append(hashX)
return pop
def _evaluate(self, x, out, check=False, *args, **kwargs):
F = np.full((x.shape[0], self.n_obj), np.nan)
if self.problem_name == 'nas101':
benchmark_api = kwargs['algorithm'].benchmark_api
for i in range(x.shape[0]):
cell = api.ModelSpec(matrix=np.array(x[i][:-1], dtype=np.int),
ops=x[i][-1].tolist())
module_hash = benchmark_api.get_module_hash(cell)
F[i, 0] = (self.benchmark_data[module_hash]['params'] -
self.min_max['min_model_params']) / (
self.min_max['max_model_params'] -
self.min_max['min_model_params'])
F[i, 1] = 1 - self.benchmark_data[module_hash]['val_acc']
self._n_evaluated += 1
elif self.problem_name == 'cifar10' or self.problem_name == 'cifar100':
for i in range(x.shape[0]):
x_str = ''.join(x[i].tolist())
F[i, 0] = (self.benchmark_data[x_str]['MMACs'] -
self.min_max['min_MMACs']) / (
self.min_max['max_MMACs'] -
self.min_max['min_MMACs'])
F[i, 1] = 1 - self.benchmark_data[x_str]['val_acc'] / 100
self._n_evaluated += 1
out['F'] = F
if check:
return F
def _sampling(self, n_samples):
P = Population(n_samples)
P_hashX = []
i = 0
if BENCHMARK_NAME == 'NAS-Bench-101':
while i < n_samples:
matrix_2D, ops_STRING = create_model()
MS = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING)
if BENCHMARK_API.is_valid(MS):
hashX = BENCHMARK_API.get_module_hash(MS)
if hashX not in P_hashX:
P_hashX.append(hashX)
matrix_1D = decoding_matrix(matrix_2D)
ops_INT = decoding_ops(ops_STRING)
X = combine_matrix1D_and_opsINT(matrix=matrix_1D,
ops=ops_INT)
F, _ = self.evaluate(X=X, using_surrogate_model=False)
P[i].set('X', X)
P[i].set('hashX', hashX)
P[i].set('F', F)
self.update_A(P[i])
i += 1
else:
if BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100'\
or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120':
l = 6
opt = ['0', '1', '2', '3', '4']
else:
l = 14
opt = ['I', '1', '2']
while i < n_samples:
X = np.random.choice(opt, l)
hashX = convert_to_hashX(X, BENCHMARK_NAME)
if hashX not in P_hashX:
P_hashX.append(hashX)
F, _ = self.evaluate(X=X)
P[i].set('X', X)
P[i].set('hashX', hashX)
P[i].set('F', F)
self.update_A(P[i])
i += 1
return P
def encode_(X):
hashX = []
for x in X:
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(x)
matrix_2D = encoding_matrix(matrix_1D)
ops_STRING = encoding_ops(ops_INT)
modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING)
hashX.append(BENCHMARK_API.get_module_hash(modelspec))
return np.array(hashX)
def _do_each_gen(self, first=False):
if self.using_surrogate_model:
self.alpha = np.mean(self.F_total)
if self.m_nEs - self.nEs < 2 * self.m_nEs // 3 or self.n_updates == 15:
self.update_model = False
if not first:
tmp_set_X = np.array(self.tmp_A_X)
tmp_set_hashX = np.array(self.tmp_A_hashX)
tmp_set = Population(len(self.tmp_A_X))
tmp_set.set('X', tmp_set_X)
tmp_set.set('hashX', tmp_set_hashX)
for i in range(len(tmp_set)):
if (tmp_set[i].get('hashX')
not in self.A_hashX) and (tmp_set[i].get('hashX')
not in self.DS):
F, _ = self.evaluate(tmp_set[i].get('X'),
using_surrogate_model=False,
count_nE=True)
tmp_set[i].set('F', F)
self.update_A(tmp_set[i])
if self.n_gen % self.update_model_after_n_gens == 0:
data = np.array(self.training_data)
self.training_data = []
X = []
Y = []
checked = []
for i in range(len(data)):
if BENCHMARK_NAME == 'NAS-Bench-101':
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(
data[i].get('X'))
matrix_2D = encoding_matrix(matrix_1D)
ops_STRING = encoding_ops(ops_INT)
modelspec = api.ModelSpec(matrix=matrix_2D,
ops=ops_STRING)
hashX = BENCHMARK_API.get_module_hash(modelspec)
else:
hashX = convert_to_hashX(data[i].get('X'),
BENCHMARK_NAME)
if (hashX not in checked) and (hashX not in self.DS) and (
hashX not in self.A_hashX):
checked.append(hashX)
F, _ = self.evaluate(data[i].get('X'),
using_surrogate_model=False,
count_nE=True)
data[i].set('F', F)
self.update_A(data[i])
X.append(data[i].get('X'))
Y.append(F[1])
for i in range(len(self.A_X)):
X.append(self.A_X[i])
Y.append(self.A_F[i][1])
X = np.array(X)
Y = np.array(Y)
if self.update_model:
self.n_updates += 1
if BENCHMARK_NAME == 'NAS-Bench-101':
self.surrogate_model.fit(x=X, y=Y)
else:
self.surrogate_model.fit(x=encode(X, BENCHMARK_NAME),
y=Y,
verbose=False)
if DEBUG:
print(f'Number of evaluations used: {self.nEs}/{self.m_nEs}')
if SAVE:
pf = np.array(self.A_F)
pf = np.unique(pf, axis=0)
pf = pf[np.argsort(pf[:, 0])]
pk.dump(
[pf, self.nEs],
open(
f'{self.path}/pf_eval/pf_and_evaluated_gen_{self.n_gen}.p',
'wb'))
self.worst_f0 = max(self.worst_f0, np.max(pf[:, 0]))
self.worst_f1 = max(self.worst_f1, np.max(pf[:, 1]))
dpfs = round(cal_dpfs(pareto_s=pf, pareto_front=BENCHMARK_PF_TRUE),
6)
print(self.nEs, dpfs)
if len(self.no_eval) == 0:
self.dpfs.append(dpfs)
self.no_eval.append(self.nEs)
else:
if self.nEs == self.no_eval[-1]:
self.dpfs[-1] = dpfs
else:
self.dpfs.append(dpfs)
self.no_eval.append(self.nEs)
def local_search_on_X(self, P, X, ls_on_knee_solutions=False):
P_hashX = P.get('hashX')
len_soln = len(P_hashX[0])
S = P.new()
S = S.merge(X)
# Using for local search on knee solutions
first, last = 0, 0
if ls_on_knee_solutions:
first, last = len(S) - 2, len(S) - 1
m_searches = len_soln
if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100':
ops = ['I', '1', '2']
else:
ops = ['0', '1', '2', '3', '4']
for i in range(len(S)):
# Avoid stuck because don't find any better architecture
tmp_m_searches = 100
tmp_n_searches = 0
checked = [S[i].get('hashX')]
n_searches = 0
while (n_searches < m_searches) and (tmp_n_searches <
tmp_m_searches):
tmp_n_searches += 1
o = S[i].copy() # --> neighboring solution
if BENCHMARK_NAME == 'NAS-Bench-101':
''' Local search on edges '''
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(
o.get('X'))
ops_STRING = encoding_ops(ops_INT)
while True:
idxs = np.random.choice(range(len(matrix_1D)),
size=self.n_vars,
replace=False)
matrix_1D_new = matrix_1D.copy()
matrix_1D_new[idxs] = 1 - matrix_1D[idxs]
matrix_2D_new = encoding_matrix(matrix_1D_new)
modelspec = api.ModelSpec(matrix=matrix_2D_new,
ops=ops_STRING)
if BENCHMARK_API.is_valid(modelspec):
o_hashX = BENCHMARK_API.get_module_hash(modelspec)
o_X = combine_matrix1D_and_opsINT(
matrix_1D_new, ops_INT)
break
else:
idxs = np.random.choice(range(len(o.get('X'))),
size=self.n_vars,
replace=False)
o_X = o.get('X').copy()
for idx in idxs:
allowed_ops = ops.copy()
allowed_ops.remove(o.get('X')[idx])
new_op = np.random.choice(allowed_ops)
o_X[idx] = new_op
o_hashX = convert_to_hashX(o_X, BENCHMARK_NAME)
if (o_hashX not in checked) and (o_hashX not in P_hashX) and (
o_hashX not in self.DS):
checked.append(o_hashX)
n_searches += 1
o_F, twice = self.evaluate(
o_X, using_surrogate_model=self.using_surrogate_model)
if i == first and LOCAL_SEARCH_ON_KNEE_SOLUTIONS:
better_idv = find_better_idv(o_F, S[i].get('F'),
'first')
elif i == last and LOCAL_SEARCH_ON_KNEE_SOLUTIONS:
better_idv = find_better_idv(o_F, S[i].get('F'),
'last')
else:
better_idv = find_better_idv(o_F, S[i].get('F'))
if better_idv == 1: # --> neighboring solutions is better
S[i].set('X', o_X)
S[i].set('hashX', o_hashX)
S[i].set('F', o_F)
if not self.using_surrogate_model:
self.update_A(S[i])
else:
if twice:
self.update_A(S[i])
else:
self.training_data.append(S[i])
self.update_fake_A(S[i])
else: # --> no one is better || current solution is better
o.set('X', o_X)
o.set('hashX', o_hashX)
o.set('F', o_F)
if not self.using_surrogate_model:
self.update_A(o)
else:
if twice:
self.update_A(o)
else:
self.training_data.append(o)
self.update_fake_A(o)
return S
def _mutation(self, P, O):
P_hashX = P.get('hashX')
new_O = Population(len(O))
new_O_hashX = []
old_O_X = O.get('X')
i = 0
full = False
pM = 1 / len(old_O_X[0])
while not full:
if BENCHMARK_NAME == 'NAS-Bench-101':
for x in old_O_X:
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(x)
new_matrix_1D = matrix_1D.copy()
new_ops_INT = ops_INT.copy()
pM_idxs_matrix = np.random.rand(len(new_matrix_1D))
for j in range(len(pM_idxs_matrix)):
if pM_idxs_matrix[j] <= pM:
new_matrix_1D[j] = 1 - new_matrix_1D[j]
pM_idxs_ops = np.random.rand(len(new_ops_INT))
for j in range(len(pM_idxs_ops)):
if pM_idxs_ops[j] <= pM:
choices = [0, 1, 2]
choices.remove(new_ops_INT[j])
new_ops_INT[j] = np.random.choice(choices)
matrix_2D = encoding_matrix(new_matrix_1D)
ops_STRING = encoding_ops(new_ops_INT)
new_MS = api.ModelSpec(matrix_2D, ops_STRING)
if BENCHMARK_API.is_valid(new_MS):
hashX = BENCHMARK_API.get_module_hash(new_MS)
if (hashX not in new_O_hashX) and (
hashX not in P_hashX) and (hashX
not in self.DS):
X = combine_matrix1D_and_opsINT(
new_matrix_1D, new_ops_INT)
new_O_hashX.append(hashX)
F, twice = self.evaluate(
X=X,
using_surrogate_model=self.
using_surrogate_model)
new_O[i].set('X', X)
new_O[i].set('hashX', hashX)
new_O[i].set('F', F)
if not self.using_surrogate_model:
self.update_A(new_O[i])
else:
if twice:
self.update_A(new_O[i])
else:
self.training_data.append(new_O[i])
self.update_fake_A(new_O[i])
i += 1
if i == len(P):
full = True
break
else:
if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100':
opt = ['I', '1', '2']
else:
opt = ['0', '1', '2', '3', '4']
pM_idxs = np.random.rand(old_O_X.shape[0], old_O_X.shape[1])
for m in range(len(old_O_X)):
X = old_O_X[m].copy()
for n in range(pM_idxs.shape[1]):
if pM_idxs[m][n] <= pM:
allowed_opt = opt.copy()
allowed_opt.remove(X[n])
X[n] = np.random.choice(allowed_opt)
hashX = convert_to_hashX(X, BENCHMARK_NAME)
if (hashX not in new_O_hashX) and (
hashX not in P_hashX) and (hashX not in self.DS):
new_O_hashX.append(hashX)
F, twice = self.evaluate(
X=X,
using_surrogate_model=self.using_surrogate_model)
new_O[i].set('X', X)
new_O[i].set('hashX', hashX)
new_O[i].set('F', F)
if not self.using_surrogate_model:
self.update_A(new_O[i])
else:
if twice:
self.update_A(new_O[i])
else:
self.training_data.append(new_O[i])
self.update_fake_A(new_O[i])
i += 1
if i == len(P):
full = True
break
return new_O
def _crossover(self, P, pC=0.9):
O = Population(len(P))
O_hashX = []
nCOs = 0 # --> Avoid to stuck
i = 0
full = False
while not full:
idx = np.random.choice(len(P),
size=(len(P) // 2, 2),
replace=False)
P_ = P[idx]
if BENCHMARK_NAME == 'NAS-Bench-101':
for j in range(len(P_)):
if np.random.random() < pC:
new_O1_X, new_O2_X = crossover(P_[j][0].get('X'),
P_[j][1].get('X'),
self.typeC)
matrix1_1D, ops1_INT = split_to_matrix1D_and_opsINT(
new_O1_X)
matrix2_1D, ops2_INT = split_to_matrix1D_and_opsINT(
new_O2_X)
matrix1_2D = encoding_matrix(matrix1_1D)
matrix2_2D = encoding_matrix(matrix2_1D)
ops1_STRING = encoding_ops(ops1_INT)
ops2_STRING = encoding_ops(ops2_INT)
new_MS1 = api.ModelSpec(matrix=matrix1_2D,
ops=ops1_STRING)
new_MS2 = api.ModelSpec(matrix=matrix2_2D,
ops=ops2_STRING)
new_MS_lst = [new_MS1, new_MS2]
new_O_X_lst = [new_O1_X, new_O2_X]
for m in range(2):
if BENCHMARK_API.is_valid(new_MS_lst[m]):
new_O_hashX = BENCHMARK_API.get_module_hash(
new_MS_lst[m])
if nCOs <= 100:
if (new_O_hashX not in O_hashX) and (
new_O_hashX not in self.DS):
O_hashX.append(new_O_hashX)
new_O_F, twice = self.evaluate(
X=new_O_X_lst[m],
using_surrogate_model=self.
using_surrogate_model)
O[i].set('X', new_O_X_lst[m])
O[i].set('hashX', new_O_hashX)
O[i].set('F', new_O_F)
if not self.using_surrogate_model:
self.update_A(O[i])
else:
if twice:
self.update_A(O[i])
else:
self.training_data.append(O[i])
self.update_fake_A(O[i])
i += 1
if i == len(P):
full = True
break
else:
O_hashX.append(new_O_hashX)
new_O_F, twice = self.evaluate(
X=new_O_X_lst[m],
using_surrogate_model=self.
using_surrogate_model)
O[i].set('X', new_O_X_lst[m])
O[i].set('hashX', new_O_hashX)
O[i].set('F', new_O_F)
if not self.using_surrogate_model:
self.update_A(O[i])
else:
if twice:
self.update_A(O[i])
else:
self.training_data.append(O[i])
self.update_fake_A(O[i])
i += 1
if i == len(P):
full = True
break
else:
for m in range(2):
O[i].set('X', P_[j][m].get('X'))
O[i].set('hashX', P_[j][m].get('hashX'))
O[i].set('F', P_[j][m].get('F'))
i += 1
if i == len(P):
full = True
break
if full:
break
else:
for j in range(len(P_)):
if np.random.random() < pC:
o1_X, o2_X = crossover(P_[j][0].get('X'),
P_[j][1].get('X'), self.typeC)
o_X = [o1_X, o2_X]
o_hashX = [
convert_to_hashX(o1_X, BENCHMARK_NAME),
convert_to_hashX(o2_X, BENCHMARK_NAME)
]
if nCOs <= 100:
for m in range(2):
if (o_hashX[m]
not in O_hashX) and (o_hashX[m]
not in self.DS):
O_hashX.append(o_hashX[m])
o_F, twice = self.evaluate(
X=o_X[m],
using_surrogate_model=self.
using_surrogate_model)
O[i].set('X', o_X[m])
O[i].set('hashX', o_hashX[m])
O[i].set('F', o_F)
if not self.using_surrogate_model:
self.update_A(O[i])
else:
if twice:
self.update_A(O[i])
else:
self.training_data.append(O[i])
self.update_fake_A(O[i])
i += 1
if i == len(P):
full = True
break
else:
for m in range(2):
O_hashX.append(o_hashX[m])
o_F, twice = self.evaluate(
X=o_X[m],
using_surrogate_model=self.
using_surrogate_model)
O[i].set('X', o_X[m])
O[i].set('hashX', o_hashX[m])
O[i].set('F', o_F)
if not self.using_surrogate_model:
self.update_A(O[i])
else:
if twice:
self.update_A(O[i])
else:
self.training_data.append(O[i])
self.update_fake_A(O[i])
i += 1
if i == len(P):
full = True
break
else:
for m in range(2):
O[i].set('X', P_[j][m].get('X'))
O[i].set('hashX', P_[j][m].get('hashX'))
O[i].set('F', P_[j][m].get('F'))
i += 1
if i == len(P):
full = True
break
if full:
break
nCOs += 1
return O
def evaluate(self, X, using_surrogate_model=False, count_nE=True):
F = np.full(2, fill_value=np.nan)
twice = False # --> using on 'surrogate model method'
if not using_surrogate_model:
if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100':
hashX = ''.join(X.tolist())
F[0] = np.round(
(BENCHMARK_DATA[hashX]['MMACs'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'] / 100, 6)
elif BENCHMARK_NAME == 'NAS-Bench-101':
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(X)
matrix_2D = encoding_matrix(matrix_1D)
ops_STRING = encoding_ops(ops_INT)
modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING)
hashX = BENCHMARK_API.get_module_hash(modelspec)
F[0] = np.round(
(BENCHMARK_DATA[hashX]['params'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'], 6)
elif BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100' \
or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120':
hashX = convert_to_hashX(X, BENCHMARK_NAME)
F[0] = np.round(
(BENCHMARK_DATA[hashX]['FLOP'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = np.round(
1 - BENCHMARK_DATA[hashX]['test-accuracy'] / 100, 6)
if count_nE:
self.nEs += 1
self.F_total.append(F[1])
else:
if BENCHMARK_NAME == 'MacroNAS-CIFAR-10' or BENCHMARK_NAME == 'MacroNAS-CIFAR-100':
encode_X = encode(X, BENCHMARK_NAME)
hashX = ''.join(X.tolist())
F[0] = np.round(
(BENCHMARK_DATA[hashX]['MMACs'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = self.surrogate_model.predict(np.array([encode_X]))[0][0]
if F[1] < self.alpha:
twice = True
F[1] = np.round(1 - BENCHMARK_DATA[hashX]['val_acc'] / 100,
6)
self.nEs += 1
elif BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-10' or BENCHMARK_NAME == 'NAS-Bench-201-CIFAR-100' \
or BENCHMARK_NAME == 'NAS-Bench-201-ImageNet16-120':
hashX = convert_to_hashX(X, BENCHMARK_NAME)
encode_X = encode(X, BENCHMARK_NAME)
F[0] = np.round(
(BENCHMARK_DATA[hashX]['FLOP'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = self.surrogate_model.predict(np.array([encode_X]))[0][0]
if F[1] < self.alpha:
twice = True
F[1] = np.round(
1 - BENCHMARK_DATA[hashX]['test-accuracy'] / 100, 6)
self.nEs += 1
else:
matrix_1D, ops_INT = split_to_matrix1D_and_opsINT(X)
matrix_2D = encoding_matrix(matrix_1D)
ops_STRING = encoding_ops(ops_INT)
modelspec = api.ModelSpec(matrix=matrix_2D, ops=ops_STRING)
hashX = BENCHMARK_API.get_module_hash(modelspec)
F[0] = np.round(
(BENCHMARK_DATA[hashX]['params'] - BENCHMARK_MIN_MAX[0]) /
(BENCHMARK_MIN_MAX[1] - BENCHMARK_MIN_MAX[0]), 6)
F[1] = self.surrogate_model.predict(np.array([X]))[0][0]
if F[1] < self.alpha:
twice = True
F[1] = 1 - BENCHMARK_DATA[hashX]['val_acc']
self.nEs += 1
if twice:
self.F_total.append(F[1])
return F, twice
def get_model_spec(X):
edges_matrix, ops_matrix = X2matrices(X)
return api.ModelSpec(edges_matrix, ops_matrix)