def __init__(self, env, features, labels, idx_train, idx_val, idx_test, list_action_space, ratio, reward_type='binary', batch_size=30, num_wrong=0, bilin_q=1, embed_dim=64, gm='mean_field', mlp_hidden=64, max_lv=1, save_dir='checkpoint_dqn', device=None): assert device is not None, "'device' cannot be None, please specify it" self.features = features self.labels = labels self.possible_labels = torch.arange(labels.max() + 1).to(labels.device) self.idx_train = idx_train self.idx_val = idx_val self.idx_test = idx_test self.num_wrong = num_wrong self.list_action_space = list_action_space degrees = np.array([len(d) for n, d in list_action_space.items()]) N = len(degrees[degrees > 0]) self.n_injected = len(degrees) - N assert self.n_injected == int(ratio * N) self.injected_nodes = np.arange(N)[-self.n_injected: ] self.reward_type = reward_type self.batch_size = batch_size self.save_dir = save_dir if not osp.exists(save_dir): os.system('mkdir -p %s' % save_dir) self.gm = gm self.device = device self.mem_pool = NstepReplayMem(memory_size=500000, n_steps=3, balance_sample=reward_type == 'binary', model='nipa') self.env = env self.net = NStepQNetNode(3, features, labels, list_action_space, self.n_injected, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.old_net = NStepQNetNode(3, features, labels, list_action_space, self.n_injected, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.net = self.net.to(device) self.old_net = self.old_net.to(device) self.eps_start = 1.0 self.eps_end = 0.05 # self.eps_step = 100000 self.eps_step = 30000 self.GAMMA = 0.9 self.burn_in = 50 self.step = 0 self.pos = 0 self.best_eval = None self.take_snapshot()
def __init__(self, env, features, labels, idx_meta, idx_test, list_action_space, num_mod, reward_type, batch_size=10, num_wrong=0, bilin_q=1, embed_dim=64, gm='mean_field', mlp_hidden=64, max_lv=1, save_dir='checkpoint_dqn', device=None): assert device is not None, "'device' cannot be None, please specify it" self.features = features self.labels = labels self.idx_meta = idx_meta self.idx_test = idx_test self.num_wrong = num_wrong self.list_action_space = list_action_space self.num_mod = num_mod self.reward_type = reward_type self.batch_size = batch_size self.save_dir = save_dir if not osp.exists(save_dir): os.system('mkdir -p {}'.format(save_dir)) self.gm = gm self.device = device self.mem_pool = NstepReplayMem(memory_size=500000, n_steps=2 * num_mod, balance_sample=reward_type == 'binary') self.env = env # self.net = QNetNode(features, labels, list_action_space) # self.old_net = QNetNode(features, labels, list_action_space) self.net = NStepQNetNode(2 * num_mod, features, labels, list_action_space, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.old_net = NStepQNetNode(2 * num_mod, features, labels, list_action_space, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.net = self.net.to(device) self.old_net = self.old_net.to(device) self.eps_start = 1.0 self.eps_end = 0.05 self.eps_step = 100000 self.burn_in = 10 self.step = 0 self.pos = 0 self.best_eval = None self.take_snapshot()
class NIPA(object): def __init__(self, env, features, labels, idx_train, idx_val, idx_test, list_action_space, ratio, reward_type='binary', batch_size=30, num_wrong=0, bilin_q=1, embed_dim=64, gm='mean_field', mlp_hidden=64, max_lv=1, save_dir='checkpoint_dqn', device=None): assert device is not None, "'device' cannot be None, please specify it" self.features = features self.labels = labels self.possible_labels = torch.arange(labels.max() + 1).to(labels.device) self.idx_train = idx_train self.idx_val = idx_val self.idx_test = idx_test self.num_wrong = num_wrong self.list_action_space = list_action_space degrees = np.array([len(d) for n, d in list_action_space.items()]) N = len(degrees[degrees > 0]) self.n_injected = len(degrees) - N assert self.n_injected == int(ratio * N) self.injected_nodes = np.arange(N)[-self.n_injected:] self.reward_type = reward_type self.batch_size = batch_size self.save_dir = save_dir if not osp.exists(save_dir): os.system('mkdir -p %s' % save_dir) self.gm = gm self.device = device self.mem_pool = NstepReplayMem(memory_size=500000, n_steps=3, balance_sample=reward_type == 'binary', model='nipa') self.env = env self.net = NStepQNetNode(3, features, labels, list_action_space, self.n_injected, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.old_net = NStepQNetNode(3, features, labels, list_action_space, self.n_injected, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.net = self.net.to(device) self.old_net = self.old_net.to(device) self.eps_start = 1.0 self.eps_end = 0.05 # self.eps_step = 100000 self.eps_step = 30000 self.GAMMA = 0.9 self.burn_in = 50 self.step = 0 self.pos = 0 self.best_eval = None self.take_snapshot() def take_snapshot(self): self.old_net.load_state_dict(self.net.state_dict()) def make_actions(self, time_t, greedy=False): # TODO self.eps = self.eps_end + max( 0., (self.eps_start - self.eps_end) * (self.eps_step - max(0., self.step)) / self.eps_step) self.step += 1 if random.random() < self.eps and not greedy: actions = self.env.uniformRandActions() else: cur_state = self.env.getStateRef() # list_at = self.env.uniformRandActions() list_at = self.env.first_nodes if time_t == 1 else None actions = self.possible_actions(cur_state, list_at, time_t) actions, values = self.net(time_t, cur_state, actions, greedy_acts=True, is_inference=True) assert len(actions) == len(cur_state) # actions = list(actions.cpu().numpy()) return actions def run_simulation(self): self.env.setup() t = 0 while not self.env.isActionFinished(): list_at = self.make_actions(t) list_st = self.env.cloneState() self.env.step(list_at) assert (self.env.rewards is not None) == self.env.isActionFinished() if self.env.isActionFinished(): rewards = self.env.rewards s_prime = self.env.cloneState() else: rewards = np.zeros(len(list_at), dtype=np.float32) s_prime = self.env.cloneState() if self.env.isTerminal(): rewards = self.env.rewards s_prime = None # self.env.init_overall_steps() self.mem_pool.add_list(list_st, list_at, rewards, s_prime, [self.env.isTerminal()] * len(list_at), t) t += 1 def eval(self, training=True): self.env.init_overall_steps() self.env.setup() for _ in count(): self.env.setup() t = 0 while not self.env.isActionFinished(): list_at = self.make_actions(t, greedy=True) # print(list_at) self.env.step(list_at, inference=True) t += 1 if self.env.isTerminal(): break device = self.labels.device extra_adj = self.env.modified_list[0].get_extra_adj(device=device) adj = self.env.classifier.norm_tool.norm_extra(extra_adj) labels = torch.cat((self.labels, self.env.modified_label_list[0])) self.env.classifier.fit(self.features, adj, labels, self.idx_train, self.idx_val, normalize=False, patience=50) output = self.env.classifier(self.features, adj) loss, acc = loss_acc(output, self.labels, self.idx_test) print('\033[93m average test: acc %.5f\033[0m' % (acc)) if training == True and self.best_eval is None or acc < self.best_eval: print( '----saving to best attacker since this is the best attack rate so far.----' ) torch.save(self.net.state_dict(), osp.join(self.save_dir, 'epoch-best.model')) with open(osp.join(self.save_dir, 'epoch-best.txt'), 'w') as f: f.write('%.4f\n' % acc) # with open(osp.join(self.save_dir, 'attack_solution.txt'), 'w') as f: # for i in range(len(self.idx_meta)): # f.write('%d: [' % self.idx_meta[i]) # for e in self.env.modified_list[i].directed_edges: # f.write('(%d %d)' % e) # f.write('] succ: %d\n' % (self.env.binary_rewards[i])) self.best_eval = acc def train(self, num_episodes=10, lr=0.01): optimizer = optim.Adam(self.net.parameters(), lr=lr) self.env.init_overall_steps() pbar = tqdm(range(self.burn_in), unit='batch') for p in pbar: self.run_simulation() self.mem_pool.print_count() for i_episode in tqdm(range(num_episodes)): self.env.init_overall_steps() for t in count(): self.run_simulation() cur_time, list_st, list_at, list_rt, list_s_primes, list_term = self.mem_pool.sample( batch_size=self.batch_size) list_target = torch.Tensor(list_rt).to(self.device) if not list_term[0]: actions = self.possible_actions(list_st, list_at, cur_time + 1) _, q_rhs = self.old_net(cur_time + 1, list_s_primes, actions, greedy_acts=True) list_target += self.GAMMA * q_rhs # list_target = list_target.view(-1, 1) _, q_sa = self.net(cur_time, list_st, list_at) loss = F.mse_loss(q_sa, list_target) loss = torch.clamp(loss, -1, 1) optimizer.zero_grad() loss.backward() # print([x[0] for x in self.nnamed_parameters() if x[1].grad is None]) # for param in self.net.parameters(): # if param.grad is None: # continue # param.grad.data.clamp_(-1, 1) optimizer.step() # pbar.set_description('eps: %.5f, loss: %0.5f, q_val: %.5f' % (self.eps, loss, torch.mean(q_sa)) ) if t % 20 == 0: print( 'eps: %.5f, loss: %0.5f, q_val: %.5f, list_target: %.5f' % (self.eps, loss, torch.mean(q_sa), torch.mean(list_target))) if self.env.isTerminal(): break # if (t+1) % 50 == 0: # self.take_snapshot() if i_episode % 1 == 0: self.take_snapshot() if i_episode % 1 == 0: self.eval() def possible_actions(self, list_st, list_at, t): ''' list_st: current state list_at: current action return: actions for next state ''' t = t % 3 if t == 0: return np.tile(self.injected_nodes, ((len(list_st), 1))) if t == 1: actions = [] for i in range(len(list_at)): a_prime = list_st[i][0].get_possible_nodes(list_at[i]) actions.append(a_prime) return actions if t == 2: return self.possible_labels.repeat((len(list_st), 1))
class Agent(object): def __init__(self, env, features, labels, idx_meta, idx_test, list_action_space, num_mod, reward_type, batch_size=10, num_wrong=0, bilin_q=1, embed_dim=64, gm='mean_field', mlp_hidden=64, max_lv=1, save_dir='checkpoint_dqn', device=None): assert device is not None, "'device' cannot be None, please specify it" self.features = features self.labels = labels self.idx_meta = idx_meta self.idx_test = idx_test self.num_wrong = num_wrong self.list_action_space = list_action_space self.num_mod = num_mod self.reward_type = reward_type self.batch_size = batch_size self.save_dir = save_dir if not osp.exists(save_dir): os.system('mkdir -p {%s}' % save_dir) self.gm = gm self.device = device self.mem_pool = NstepReplayMem(memory_size=500000, n_steps=2 * num_mod, balance_sample=reward_type == 'binary') self.env = env # self.net = QNetNode(features, labels, list_action_space) # self.old_net = QNetNode(features, labels, list_action_space) self.net = NStepQNetNode(2 * num_mod, features, labels, list_action_space, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.old_net = NStepQNetNode(2 * num_mod, features, labels, list_action_space, bilin_q=bilin_q, embed_dim=embed_dim, mlp_hidden=mlp_hidden, max_lv=max_lv, gm=gm, device=device) self.net = self.net.to(device) self.old_net = self.old_net.to(device) self.eps_start = 1.0 self.eps_end = 0.05 self.eps_step = 100000 self.burn_in = 10 self.step = 0 self.pos = 0 self.best_eval = None self.take_snapshot() def take_snapshot(self): self.old_net.load_state_dict(self.net.state_dict()) def make_actions(self, time_t, greedy=False): self.eps = self.eps_end + max( 0., (self.eps_start - self.eps_end) * (self.eps_step - max(0., self.step)) / self.eps_step) if random.random() < self.eps and not greedy: actions = self.env.uniformRandActions() else: cur_state = self.env.getStateRef() actions, values = self.net(time_t, cur_state, None, greedy_acts=True, is_inference=True) actions = list(actions.cpu().numpy()) return actions def run_simulation(self): if (self.pos + 1) * self.batch_size > len(self.idx_test): self.pos = 0 random.shuffle(self.idx_test) selected_idx = self.idx_test[self.pos * self.batch_size:(self.pos + 1) * self.batch_size] self.pos += 1 self.env.setup(selected_idx) t = 0 list_of_list_st = [] list_of_list_at = [] while not self.env.isTerminal(): list_at = self.make_actions(t) list_st = self.env.cloneState() self.env.step(list_at) # TODO Wei added line #87 env = self.env assert (env.rewards is not None) == env.isTerminal() if env.isTerminal(): rewards = env.rewards s_prime = None else: rewards = np.zeros(len(list_at), dtype=np.float32) s_prime = self.env.cloneState() self.mem_pool.add_list(list_st, list_at, rewards, s_prime, [env.isTerminal()] * len(list_at), t) list_of_list_st.append(deepcopy(list_st)) list_of_list_at.append(deepcopy(list_at)) t += 1 # if the reward type is nll_loss, directly return if self.reward_type == 'nll': return T = t cands = self.env.sample_pos_rewards(len(selected_idx)) if len(cands): for c in cands: sample_idx, target = c doable = True for t in range(T): if self.list_action_space[target] is not None and ( not list_of_list_at[t][sample_idx] in self.list_action_space[target]): doable = False # TODO WHY False? This is only 1-hop neighbour break if not doable: continue for t in range(T): s_t = list_of_list_st[t][sample_idx] a_t = list_of_list_at[t][sample_idx] s_t = [target, deepcopy(s_t[1]), s_t[2]] if t + 1 == T: s_prime = (None, None, None) r = 1.0 term = True else: s_prime = list_of_list_st[t + 1][sample_idx] s_prime = [target, deepcopy(s_prime[1]), s_prime[2]] r = 0.0 term = False self.mem_pool.mem_cells[t].add(s_t, a_t, r, s_prime, term) def eval(self, training=True): self.env.setup(self.idx_meta) t = 0 while not self.env.isTerminal(): list_at = self.make_actions(t, greedy=True) self.env.step(list_at) t += 1 acc = 1 - (self.env.binary_rewards + 1.0) / 2.0 acc = np.sum(acc) / (len(self.idx_meta) + self.num_wrong) print('\033[93m average test: acc %.5f\033[0m' % (acc)) if training == True and self.best_eval is None or acc < self.best_eval: print( '----saving to best attacker since this is the best attack rate so far.----' ) torch.save(self.net.state_dict(), osp.join(self.save_dir, 'epoch-best.model')) with open(osp.join(self.save_dir, 'epoch-best.txt'), 'w') as f: f.write('%.4f\n' % acc) with open(osp.join(self.save_dir, 'attack_solution.txt'), 'w') as f: for i in range(len(self.idx_meta)): f.write('%d: [' % self.idx_meta[i]) for e in self.env.modified_list[i].directed_edges: f.write('(%d %d)' % e) f.write('] succ: %d\n' % (self.env.binary_rewards[i])) self.best_eval = acc def train(self, num_steps=100000, lr=0.001): pbar = tqdm(range(self.burn_in), unit='batch') for p in pbar: self.run_simulation() pbar = tqdm(range(num_steps), unit='steps') optimizer = optim.Adam(self.net.parameters(), lr=lr) for self.step in pbar: self.run_simulation() if self.step % 123 == 0: # update the params of old_net self.take_snapshot() if self.step % 500 == 0: self.eval() cur_time, list_st, list_at, list_rt, list_s_primes, list_term = self.mem_pool.sample( batch_size=self.batch_size) list_target = torch.Tensor(list_rt).to(self.device) if not list_term[0]: target_nodes, _, picked_nodes = zip(*list_s_primes) _, q_t_plus_1 = self.old_net(cur_time + 1, list_s_primes, None) _, q_rhs = node_greedy_actions(target_nodes, picked_nodes, q_t_plus_1, self.old_net) list_target += q_rhs # list_target = Variable(list_target.view(-1, 1)) list_target = list_target.view(-1, 1) _, q_sa = self.net(cur_time, list_st, list_at) q_sa = torch.cat(q_sa, dim=0) loss = F.mse_loss(q_sa, list_target) optimizer.zero_grad() loss.backward() optimizer.step() pbar.set_description('eps: %.5f, loss: %0.5f, q_val: %.5f' % (self.eps, loss, torch.mean(q_sa)))