class Experiment_struct: def __init__(self, nn, sample_size=5, budget=20, positive_num=2, r_p=0.99, uncertain_bit=3, add_num=20000): self.nn = nn self.spl = Sampler_struct(nn) self.opt = Optimizer(self.spl.get_dim(), self.spl.get_parametets_subscript()) # sample_size = 5 # the instance number of sampling in an iteration # budget = 20 # budget in online style # positive_num = 2 # the set size of PosPop # r_p = 0.99 # the probability of sample in model # uncertain_bit = 3 # the dimension size that is sampled randomly # set hyper-parameter for optimization, budget is useless for single step optimization self.opt.set_parameters(ss=sample_size, bud=budget, pn=positive_num, rp=r_p, ub=uncertain_bit) # clear optimization model self.opt.clear() self.budget = budget pros = self.opt.sample() self.spl.renewp(pros) self.eva = Evaluater() self.eva.add_data(add_num) self.opt_p_log = [] print(self.eva.max_steps) print(len(pros)) for i in range(self.budget): self.opt_p_log.append(pros) spl_list = self.spl.sample() self.nn.cell_list.append(spl_list) # score = np.random.random() # time_tmp = time.time() # score = self.eva.evaluate(self.nn, i, time_tmp) score = self.eva.evaluate(self.nn) # Updating optimization based on the obtained scores # Upadting pros in spl self.opt.update_model(pros, -score) pros = self.opt.sample() self.spl.renewp(pros) self.res_fea = self.opt.get_optimal().get_features() self.res_fit = self.opt.get_optimal().get_fitness() print('best:') print('features', self.res_fea) # pros print('fitness', self.res_fit) # scores
class NetworkUnit: pre_block = [] def __init__(self, graph_part=[[]], cell_list=[]): self.graph_part = graph_part self.graph_full_list = [ ] # to store the structure ever used including skipping layers self.cell_list = cell_list self.score_list = [] self.spl = None self.opt = None self.table = [] def init_sample(self, pattern="Global", block_num=0): from optimizer import Optimizer if pattern == "Block": from sampler_block import Sampler self.spl = Sampler(self.graph_part, len(self.graph_part), block_num) else: from sampler_global import Sampler self.spl = Sampler(self.graph_part, len(self.graph_part)) self.opt = Optimizer(self.spl.get_dim(), self.spl.get_parametets_subscript()) self.table = [] # to store the encoding derived from opt for sampling sample_size = 3 # the instance number of sampling in an iteration budget = 20000 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 3 # the dimension size that is sampled randomly # set hyper-parameter for optimization, budget is useless for single step optimization self.opt.set_parameters(ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) # clear optimization model self.opt.clear() return
class Sampler: def __init__(self, graph_part, block_id): """ Generate adjacency of network topology. Sampling network operation based on sampled value(table). The sampling value is updated by the optimization module based on the output given by the evaluation module. Attributes: graph_part: a Network Topology(Adjacency table). block_id: The stage of neural network search. Other important operation information and parameters of optimization module are given by folder 'parameters'. """ self._p_table = [] # initializing the table value in Sampler. self._graph_part = graph_part self._node_number = len(self._graph_part) self._pattern = NAS_CONFIG['nas_main']['pattern'] # Parameter setting based on search method self._crosslayer_dis = NAS_CONFIG['spl']['skip_max_dist'] + 1 # dis control self._cross_node_number = NAS_CONFIG['spl']['skip_max_num'] self._graph_part_invisible_node = self._graph_part_add_invisible_node() self._tmp_flag = 0 self._graph_part_invisible_node_flag = [0 for i in range(len(self._graph_part_invisible_node))] self._find_main_chain(self._graph_part_invisible_node) # print(self._graph_part_invisible_node_flag) self._crosslayer = self._get_crosslayer() # Read parameter table to get operation dictionary in stage(block_id) self._setting = self._init_setting(block_id) self._dic_index = self._init_dict() # check # Set parameters of optimization module based on the above results self.__region, self.__type = self._opt_parameters() self.__dim = Dimension() self.__dim.set_dimension_size(len(self.__region)) # 10% self.__dim.set_regions(self.__region, self.__type) self.__parameters_subscript = [] self.opt = Optimizer(self.__dim, self.__parameters_subscript) opt_para = copy.deepcopy(NAS_CONFIG["opt"]) __sample_size = opt_para["sample_size"] # the instance number of sampling in an iteration __budget = opt_para["budget"] # budget in online style __positive_num = opt_para["positive_num"] # the set size of PosPop __rand_probability = opt_para["rand_probability"] # the probability of sample in model __uncertain_bit = opt_para["uncertain_bit"] # the dimension size that is sampled randomly self.opt.set_parameters(ss=__sample_size, bud=__budget, pn=__positive_num, rp=__rand_probability, ub=__uncertain_bit) self.opt.clear() # clear optimization model def sample(self): """ Get table based on the optimization module sampling, update table in Sampler, and sample the operation configuration. No Args. Retruns: 1. cell (1d Cell list) 2. graph_full (2d int list, as NetworkItem.graph_full) 3. table (1d int list, depending on dimension) """ table = self.opt.sample() cell, graph = self.convert(table) return cell, graph, table def update_opt_model(self, table, score): """ Optimization of sampling space based on Evaluation and optimization method. Args: 1. table (1d int list, depending on dimension) 2. score(float, 0 ~ 1.0) No returns. """ self.opt.update_model(table, score) # here "-" represent that we minimize the loss def _init_setting(self, block_id): _setting_tmp = collections.OrderedDict() _setting_tmp = copy.deepcopy(NAS_CONFIG['spl']['space']) if NAS_CONFIG['spl']['pool_switch'] == 0 and 'pooling' in _setting_tmp: del _setting_tmp['pooling'] for key in _setting_tmp: for op in _setting_tmp[key]: if type(_setting_tmp[key][op][0]) is list: _setting_tmp[key][op] = _setting_tmp[key][op][block_id] return _setting_tmp def _graph_part_add_invisible_node(self): graph_part_tmp = [] for i in self._graph_part: if not i: graph_part_tmp.append([self._node_number]) else: graph_part_tmp.append(i) graph_part_tmp.append([]) return graph_part_tmp def _find_main_chain(self, graph): q = Queue() q.put([0, 0]) ma = 0 while q.empty() is False: f = q.get() ma = max(f[1], ma) for i in self._graph_part_invisible_node[f[0]]: q.put([i, f[1]+1]) self._graph_part_invisible_node_flag[0] = 1 self._dfs(0, 0, ma) return def _dfs(self, node_id, cnt, ma): if cnt == ma: self._tmp_flag = 1 if self._tmp_flag == 1: return for i in self._graph_part_invisible_node[node_id]: if self._graph_part_invisible_node_flag[i] == 0 and self._tmp_flag == 0: self._graph_part_invisible_node_flag[i] = 1 self._dfs(i, cnt+1, ma) if self._tmp_flag == 0: self._graph_part_invisible_node_flag[i] = 0 def _get_crosslayer(self): """ utilizing breadth-first search to set the possible cross layer connection for each node of the network topology. """ cl = [] for i in range(self._node_number): if self._graph_part_invisible_node_flag[i] == 1: cl.append(self._bfs(i)) else: cl.append([]) return cl def _bfs(self, node_id): res_list = [] q = Queue() q.put([node_id, 0]) v = [] for i in range(self._node_number + 1): v.append(0) v[node_id] = 1 while q.empty() is False: f = q.get() if f[1] >= 2: if f[1] <= self._crosslayer_dis: res_list.append(f[0]) else: continue # for j in self._graph_part[f[0]]: for j in self._graph_part_invisible_node[f[0]]: if self._graph_part_invisible_node_flag[j] == 1: if v[j] == 0: q.put([j, f[1] + 1]) v[j] = 1 return res_list # def _region_cross_type(self, __region_tmp, __type_tmp, i): region_tmp = copy.copy(__region_tmp) type_tmp = copy.copy(__type_tmp) for j in range(self._cross_node_number): if self._graph_part_invisible_node_flag[i] == 1: region_tmp.append([0, len(self._crosslayer[i])]) type_tmp.append(2) return region_tmp, type_tmp def _opt_parameters(self): """Get the parameters of optimization module based on parameter document.""" __type_tmp = [] __region_tmp = [] for i in range(len(self._dic_index)): __region_tmp.append([0, 0]) # __region_tmp = [[0, 0] for _ in range(len(self._dic_index))] for key in self._dic_index: tmp = int(self._dic_index[key][1] - self._dic_index[key][0]) __region_tmp[self._dic_index[key][-1]] = [0, tmp] __type_tmp.append(2) __region = [] __type = [] for i in range(self._node_number): __region_cross, __type_cross = \ self._region_cross_type(__region_tmp, __type_tmp, i) __region = __region + __region_cross __type.extend(__type_cross) return __region, __type def convert(self, table_tmp): """Search corresponding operation configuration based on table.""" self._p_table = copy.deepcopy(table_tmp) res = [] l = 0 r = 0 graph_part_sample = copy.deepcopy(self._graph_part) for num in range(self._node_number): l = r r = l + len(self._dic_index) if self._graph_part_invisible_node_flag[num] == 1: r = r + self._cross_node_number p_node = self._p_table[l:r] # Take the search space of a node # print(p_node) node_cross_tmp = list(set(copy.deepcopy(p_node[len(self._dic_index):]))) for i in node_cross_tmp: if i != 0: graph_part_sample[num].append(self._crosslayer[num][i - 1]) if not graph_part_sample[num]: graph_part_sample[num].append(self._node_number) for cnt, key_type in enumerate(self._setting): if p_node[self._dic_index['type'][-1]] == cnt: tmp = (key_type,) item_list = Cell.get_format(key_type) for key_item in item_list: tmp = tmp + (self._setting[key_type][key_item] [p_node[self._dic_index[key_type + ' ' + key_item][-1]]],) # print(tmp) tmp = Cell(tmp) res.append(tmp) return res, graph_part_sample def _init_dict(self): """Operation space dictionary based on parameter file.""" dic = {} dic['type'] = (0, len(self._setting)-1, 0) cnt = 1 num = 1 for key in self._setting: for k in self._setting[key]: tmp = len(self._setting[key][k]) - 1 dic[key + ' ' + k] = (cnt, cnt + tmp, num) num += 1 cnt += tmp return dic # log def _get_cell_log(self, POOL, PATH, date): for i, j in enumerate(POOL): s = 'nn_param_' + str(i) + '_' + str(date) fp = open(PATH + s, "wb") # print(s) pickle.dump(j.cell_list, fp) def ops2table(self, ops, table_tmp): """ set the table under the output in predictor the output in predictor looks like: [['64', '7'], ['pooling'], ['64', '3'], ['256', '3'], ['1024', '1'], ['1024', '1'], ['1024', '3'], ['1024', '3'], ['1024', '3'], ['512', '1'], ['128', '5'], ['64', '3'], ['1024', '1'], ['1024', '1'], ['256', '3']] """ self._p_table = copy.deepcopy(table_tmp) table = [] l = 0 r = 0 conv_cnt = 0 pooling_cnt = 1 for i, j in enumerate(self._setting): if j == 'conv': conv_cnt = i if j == 'pooling': pooling_cnt = i for num in range(self._node_number): # Take the search space of a node l = r r = l + len(self._dic_index) if self._graph_part_invisible_node_flag[num] == 1: r = r + self._cross_node_number p_node = self._p_table[l:r] # Sample value of the current node if len(ops[num]) != 1: p_node = self._p_table[l:r] # Sample value of the current node p_node[self._dic_index['type'][-1]] = conv_cnt for j, i in enumerate(self._setting['conv']['filter_size']): if str(i) == ops[num][0]: p_node[self._dic_index['conv filter_size'][-1]] = j for j, i in enumerate(self._setting['conv']['kernel_size']): if str(i) == ops[num][1]: p_node[self._dic_index['conv kernel_size'][-1]] = j for j, i in enumerate(self._setting['conv']['activation']): if i == 'relu': p_node[self._dic_index['conv activation'][-1]] = j table = table + p_node else: if self._pattern == "Global": p_node[self._dic_index['type'][-1]] = pooling_cnt table = table + p_node return table
print('##'*40) spl = Sampler(graph_part, 50) dic_index = spl.dic_index cl = spl.crosslayer print(dic_index) # opt = Optimizer(spl.get_dim(), spl.get_parametets_subscript()) sample_size = 5 # the instance number of sampling in an iteration budget = 20000 # budget in online style positive_num = 2 # the set size of PosPop rand_probability = 0.99 # the probability of sample in model uncertain_bit = 10 # the dimension size that is sampled randomly opt.set_parameters(ss=sample_size, bud=budget, pn=positive_num, rp=rand_probability, ub=uncertain_bit) opt.clear() table = opt.sample() spl.renewp(table) __cell, graph_part_sample = spl.sample() print(__cell) # graph_part加入跨层连接的结构 print(graph_part_sample) # init 为初始化的返回,调用init_p 基于操作配置初始化进行更新 table = spl.init_p(init) spl.renewp(table) __cell, graph_part_sample = spl.sample()