def sample(self, batch_size=1, with_details=False): #num_sample if batch_size < 1: raise Exception(f'Wrong batch_size: {batch_size} < 1') inputs = torch.zeros([batch_size, self.controller_hid ]) # batch_size([1, self.controller_hid 100]) hidden = (torch.zeros([batch_size, self.controller_hid]), torch.zeros([batch_size, self.controller_hid])) if self.is_cuda: inputs = inputs.cuda() hidden = (hidden[0].cuda(), hidden[1].cuda()) entropies = [] log_probs = [] actions = [] for block_idx, action_name in enumerate(self.action_list): decoder_index = self.action_index(action_name) logits, hidden = self.forward( inputs, #Train [1, 100] derive [100,100] logits: [100,53] hidden, action_name, is_embed=(block_idx == 0)) probs = F.softmax(logits, dim=-1) #derive [100,53] log_prob = F.log_softmax(logits, dim=-1) entropy = -(log_prob * probs).sum(1, keepdim=False) action = probs.multinomial(num_samples=1).data #derive [100,1] selected_log_prob = log_prob.gather( 1, utils.get_variable(action, requires_grad=False)) entropies.append(entropy) log_probs.append(selected_log_prob[:, 0]) inputs = utils.get_variable(action[:, 0] + sum(self.num_tokens[:decoder_index]), self.is_cuda, requires_grad=False) inputs = self.encoder(inputs) actions.append(action[:, 0]) actions = torch.stack(actions).transpose(0, 1) dags = self._construct_action( actions ) #[['gat', 'max', 'tanh', 1, 128, 'cos', 'sum', 'tanh', 4, 16]] if with_details: return dags, torch.cat(log_probs), torch.cat(entropies) return dags
def sample(self, batch_size=1, with_details=False): if batch_size < 1: raise Exception('Wrong batch_size:' + str(batch_size) + '< 1') inputs = torch.zeros([batch_size, self.controller_hid]) hidden = (torch.zeros([batch_size, self.controller_hid]), torch.zeros([batch_size, self.controller_hid])) if self.is_cuda: inputs = inputs.cuda() hidden = (hidden[0].cuda(), hidden[1].cuda()) entropies = [] log_probs = [] actions = [] for block_idx, action_name in enumerate(self.action_list): decoder_index = self.action_index(action_name) logits, hidden = self.forward(inputs, hidden, action_name, is_embed=(block_idx == 0)) probs = F.softmax(logits, dim=-1) log_prob = F.log_softmax(logits, dim=-1) entropy = -(log_prob * probs).sum(1, keepdim=False) action = probs.multinomial(num_samples=1).data selected_log_prob = log_prob.gather( 1, utils.get_variable(action, requires_grad=False)) entropies.append(entropy) log_probs.append(selected_log_prob[:, 0]) inputs = utils.get_variable(action[:, 0] + sum(self.num_tokens[:decoder_index]), self.is_cuda, requires_grad=False) inputs = self.encoder(inputs) actions.append(action[:, 0]) actions = torch.stack(actions).transpose(0, 1) print('sampled actions::') print(actions) dags = construct_actions(actions, self.action_list, self.search_space) if with_details: return dags, torch.cat(log_probs), torch.cat(entropies) return dags
def train_controller(self): """ Train controller to find better structure. """ print("*" * 35, "training controller", "*" * 35) model = self.controller model.train() baseline = None adv_history = [] entropy_history = [] reward_history = [] hidden = self.controller.init_hidden(self.args.batch_size) total_loss = 0 for step in range(self.args.controller_max_step): # sample graphnas structure_list, log_probs, entropies = self.controller.sample( with_details=True) # calculate reward np_entropies = entropies.data.cpu().numpy() results = self.get_reward(structure_list, np_entropies, hidden) torch.cuda.empty_cache() if results: # has reward rewards, hidden = results else: continue # CUDA Error happens, drop structure and step into next iteration # discount if 1 > self.args.discount > 0: rewards = discount(rewards, self.args.discount) reward_history.extend(rewards) entropy_history.extend(np_entropies) # moving average baseline if baseline is None: baseline = rewards else: decay = self.args.ema_baseline_decay baseline = decay * baseline + (1 - decay) * rewards adv = rewards - baseline history.append(adv) adv = scale(adv, scale_value=0.5) adv_history.extend(adv) adv = utils.get_variable(adv, self.cuda, requires_grad=False) # policy loss loss = -log_probs * adv if self.args.entropy_mode == 'regularizer': loss -= self.args.entropy_coeff * entropies loss = loss.sum() # or loss.mean() # update self.controller_optim.zero_grad() loss.backward() if self.args.controller_grad_clip > 0: torch.nn.utils.clip_grad_norm(model.parameters(), self.args.controller_grad_clip) self.controller_optim.step() total_loss += utils.to_item(loss.data) self.controller_step += 1 torch.cuda.empty_cache() print("*" * 35, "training controller over", "*" * 35)
def init_hidden(self, batch_size): zeros = torch.zeros(batch_size, self.controller_hid) return (utils.get_variable(zeros, self.is_cuda, requires_grad=False), utils.get_variable(zeros.clone(), self.is_cuda, requires_grad=False))
def train_controller(self): """ Train controller to find better structure. """ print("*" * 35, "training controller", "*" * 35) model = self.controller model.train() baseline = None adv_history = [] entropy_history = [] reward_history = [] hidden = self.controller.init_hidden(self.args.batch_size) #2 64 100 total_loss = 0 for step in range(self.args.controller_max_step): # 100 start_time = time() # sample graphnas structure_list, log_probs, entropies = self.controller.sample( with_details=True) # num_sample = 1 #[['gat', 'max', 'tanh', 1, 128, 'cos', 'sum', 'tanh', 4, 16]] #tensor([-1.9461, -1.3946, -2.0890, -1.7695, -1.9348, -1.9490, -1.3980, -2.0886,-1.7938, -1.9401], device='cuda:0', grad_fn=<CatBackward>) #tensor([1.9459, 1.3863, 2.0794, 1.7917, 1.9458, 1.9459, 1.3862, 2.0794, 1.7917,1.9458], device='cuda:0', grad_fn=<CatBackward>) # calculate reward np_entropies = entropies.data.cpu().numpy() results = self.get_reward(structure_list, np_entropies, hidden) torch.cuda.empty_cache() if results: # has reward rewards, hidden = results else: continue # CUDA Error happens, drop structure and step into next iteration # discount if 1 > self.args.discount > 0: rewards = discount(rewards, self.args.discount) reward_history.extend(rewards) entropy_history.extend(np_entropies) # moving average baseline if baseline is None: baseline = rewards else: decay = self.args.ema_baseline_decay baseline = decay * baseline + (1 - decay) * rewards adv = rewards - baseline history.append(adv) adv = scale(adv, scale_value=0.5) adv_history.extend(adv) adv = utils.get_variable(adv, self.cuda, requires_grad=False) # policy loss loss = -log_probs * adv if self.args.entropy_mode == 'regularizer': loss -= self.args.entropy_coeff * entropies loss = loss.sum() # or loss.mean() # update self.controller_optim.zero_grad() loss.backward() if self.args.controller_grad_clip > 0: torch.nn.utils.clip_grad_norm(model.parameters(), self.args.controller_grad_clip) self.controller_optim.step() total_loss += utils.to_item(loss.data) self.controller_step += 1 torch.cuda.empty_cache() elapsed = (time() - start_time) print('[%d/%d] time %.2f ' % ( step + 1, self.args.controller_max_step, elapsed, )) print("*" * 35, "training controller over", "*" * 35)
def train_controller(self): """ Train controller to find better structure. """ print("*" * 35, "training controller", "*" * 35) model = self.controller model.train() baseline = None adv_history = [] entropy_history = [] reward_history = [] hidden = self.controller.init_hidden(self.args.batch_size) total_loss = 0 for step in range(self.args.controller_max_step): #contraller训练一次的时间 once_controller_train_start_time = time.time() # sample graphnas structure_list, log_probs, entropies = self.controller.sample(with_details=True) # calculate reward np_entropies = entropies.data.cpu().numpy() results = self.get_reward(structure_list, np_entropies, hidden) torch.cuda.empty_cache() if results: # has reward rewards, hidden, acc = results else: continue # discount if 1 > self.args.discount > 0: rewards = discount(rewards, self.args.discount) reward_history.extend(rewards) entropy_history.extend(np_entropies) # moving average baseline if baseline is None: baseline = rewards else: decay = self.args.ema_baseline_decay baseline = decay * baseline + (1 - decay) * rewards adv = rewards - baseline history.append(adv) adv = scale(adv, scale_value=0.5) adv_history.extend(adv) adv = utils.get_variable(adv, self.cuda, requires_grad=False) # policy loss loss = -log_probs * adv if self.args.entropy_mode == 'regularizer': loss -= self.args.entropy_coeff * entropies loss = loss.sum() # or loss.mean() # update self.controller_optim.zero_grad() loss.backward() if self.args.controller_grad_clip > 0: torch.nn.utils.clip_grad_norm(model.parameters(), self.args.controller_grad_clip) self.controller_optim.step() total_loss += utils.to_item(loss.data) self.controller_step += 1 torch.cuda.empty_cache() once_controller_train_end_time = time.time() print("the", step, "epcoh controller train time: ", once_controller_train_end_time-once_controller_train_start_time, "s") if step == 0: self.RL_train_time.append(once_controller_train_start_time) self.RL_train_time.append(once_controller_train_end_time) self.RL_train_acc.append(acc) else: self.RL_train_time.append(once_controller_train_end_time) self.RL_train_acc.append(acc) print("all RL train time list: ", self.RL_train_time) print("all RL train acc list: ", self.RL_train_acc) print("*" * 35, "training controller over", "*" * 35) experiment_data_save("controler_train.txt", self.RL_train_time, self.RL_train_acc)
def sample(self, batch_size=1, with_details=False): if batch_size < 1: raise Exception(f'Wrong batch_size: {batch_size} < 1') # LSTM带batch_size的输入x0初始化,全0矩阵 inputs = torch.zeros([batch_size, self.controller_hid]) # inputs.shape = (1, 100) # batch_size = 1, self.controller_hid = 100 # LSTM带batch_size参数的h0,c0初始化,全0矩阵 hidden = (torch.zeros([batch_size, self.controller_hid]), torch.zeros([batch_size, self.controller_hid])) # shape(hidden) = ([1,100],[1,100]) if self.is_cuda: inputs = inputs.cuda() hidden = (hidden[0].cuda(), hidden[1].cuda()) entropies = [] log_probs = [] actions = [] for block_idx, action_name in enumerate(self.action_list): decoder_index = self.action_index(action_name) logits, hidden = self.forward(inputs, hidden, action_name, is_embed=(block_idx == 0)) probs = F.softmax(logits, dim=-1) log_prob = F.log_softmax(logits, dim=-1) entropy = -(log_prob * probs).sum(1, keepdim=False) # 基于多项式分布从probs概率向量中选取向量的索引号,该索引号对应相应action中的一个operator # 为了保证强化学习的探索性,所以没有直接选取最大的概率的operator,而是通过概率分布选取 action = probs.multinomial(num_samples=1).data # 基于action获取的索引号获取对应的log_prob(对应概率为p(x)随机变量自信息I(x)) selected_log_prob = log_prob.gather( 1, utils.get_variable(action, requires_grad=False)) # gather(log_prob,1,action) # 对log_prob按列取,取值索引号为action,取值后矩阵大小与action相同 entropies.append(entropy) log_probs.append(selected_log_prob[:, 0]) # 确定action对应的operator在embedding中的索引号 inputs = utils.get_variable(action[:, 0] + sum(self.num_tokens[:decoder_index]), self.is_cuda, requires_grad=False) # 对确定了索引号inputs进行embedding编码 inputs = self.encoder(inputs) # 将已经选择的operator索引号放入actions列表中保存 actions.append(action[:, 0]) actions = torch.stack(actions).transpose(0, 1) dags = self._construct_action(actions) if with_details: return dags, torch.cat(log_probs), torch.cat(entropies) return dags
def train_controller(self): """ Train controller to find better structure. """ print("*" * 35, "training controller", "*" * 35) model = self.controller # 使pytorch定义的controller模型进入训练模式 # 每次训练都要初始化 adv_history,entropy_history,reward_history 三个列表 model.train() baseline = None adv_history = [] entropy_history = [] reward_history = [] # 初始化带batch_size参数的中 h0,c0向量,全部为0 hidden = self.controller.init_hidden(self.args.batch_size) # self.args.batch_size = 64 # 初始化控制器LSTM模型总损失值 total_loss = 0 for step in range(self.args.controller_max_step): # self.args.controller_max_step = 100 # controller每次训练100轮,一轮选一个GNN结构 # sample graphnas structure_list, log_probs, entropies = self.controller.sample( with_details=True) # structrue_list = ['gat', 'sum', 'relu', 2, 8, 'linear', 'mlp', 'tanh', 2, 4],--->选择出的GNN结构 # log_probs = tensor([-1.9461, -1.3936, -2.0807, -1.7964, -1.9570, -1.9413, -1.3704, -2.0878, # -1.7907, -1.9185], device='cuda:0', grad_fn=<CatBackward>)--->LSTM每一步选择的operator的自信息I # entropies = tensor([1.9459, 1.3863, 2.0794, 1.7917, 1.9458, 1.9459, 1.3862, 2.0794, 1.7917, # 1.9458], device='cuda:0', grad_fn=<CatBackward>)--->LSTM每一步输出信息熵 # calculate reward np_entropies = entropies.data.cpu().numpy() results = self.get_reward(structure_list, np_entropies, hidden) # results = (rewards, hidden) hidden原封不动的回来了 torch.cuda.empty_cache() if results: # has reward rewards, hidden = results else: continue # CUDA Error happens, drop structure and step into next iteration # discount # 使用滤波器实现rewards折扣损失计算,重新计算rewards列表内的reward if 1 > self.args.discount > 0: # self.args.discount = 1 rewards = discount(rewards, self.args.discount) # 每次训练都要初始化 adv_history,entropy_history,reward_history 三个列表 # controller每次训练100轮,一轮选一个GNN结构 """" def discount(x, amount): return scipy.signal.lfilter([1], [1, -amount], x[::-1], axis=0)[::-1] x[::-1] : 将x序列翻转 """ reward_history.extend(rewards) entropy_history.extend(np_entropies) # moving average baseline if baseline is None: baseline = rewards else: decay = self.args.ema_baseline_decay baseline = decay * baseline + (1 - decay) * rewards adv = rewards - baseline history.append(adv) adv = scale(adv, scale_value=0.5) """ def scale(value, last_k=10, scale_value=1): ''' scale value into [-scale_value, scale_value], according last_k history ''' max_reward = np.max(history[-last_k:]) if max_reward == 0: return value return scale_value / max_reward * value """ adv_history.extend(adv) adv = utils.get_variable(adv, self.cuda, requires_grad=False) # policy loss loss = -log_probs * adv if self.args.entropy_mode == 'regularizer': loss -= self.args.entropy_coeff * entropies loss = loss.sum() # or loss.mean() # update self.controller_optim.zero_grad() loss.backward() if self.args.controller_grad_clip > 0: torch.nn.utils.clip_grad_norm(model.parameters(), self.args.controller_grad_clip) self.controller_optim.step() total_loss += utils.to_item(loss.data) self.controller_step += 1 torch.cuda.empty_cache() print("*" * 35, "training controller over", "*" * 35)