def sample_worker(self, pid, queue, min_batch_size): torch.randn(pid) if hasattr(self.env, 'np_random'): self.env.np_random.rand(pid) memory = Memory() logger = LoggerRL() while logger.num_steps < min_batch_size: state = self.env.reset() if self.running_state is not None: state = self.running_state(state) logger.start_episode(self.env) self.pre_episode() for t in range(10000): state_var = tensor(state).unsqueeze(0) vs_out = self.trans_policy(state_var) mean_action = self.mean_action or np.random.binomial( 1, 1 - self.noise_rate) action = self.policy_net.select_action(vs_out, mean_action)[0].numpy() action = int( action ) if self.policy_net.type == 'discrete' else action.astype( np.float64) next_state, env_reward, done, info = self.env.step(action) if self.running_state is not None: next_state = self.running_state(next_state) if self.custom_reward is not None: c_reward, c_info = self.custom_reward( self.env, state, action, info) reward = c_reward else: c_reward, c_info = 0.0, np.array([0.0]) reward = env_reward logger.step(self.env, env_reward, c_reward, c_info) mask = 0 if done else 1 exp = 1 - mean_action self.push_memory(memory, state, action, mask, next_state, reward, exp) if pid == 0 and self.render: self.env.render() if done: break state = next_state logger.end_episode(self.env) logger.end_sampling() if queue is not None: queue.put([pid, memory, logger]) else: return memory, logger
def __init__(self, model, args): self.model = model self.args = args self.iteration = 0 self.memory = Memory() if self.args.override or not os.path.isdir( self.args.output_dir) or self.args.output_dir == 'tmp': mkdir(self.args.output_dir, wipe=True) # initialize logging and model saving if self.args.output_dir is not None: self.logger = Logger( os.path.join(self.args.output_dir, 'train_log.json')) else: self.logger = Logger()
def __init__( self, n_states, n_actions, hidden_size=128, alr=2e-3, clr=2e-3, gamma=0.99, epochs=4, eps_clip=0.2, atype='cat'): self.gamma = gamma self.eps_clip = eps_clip self.epochs = epochs self.atype = atype self.memory = Memory() if atype == 'cat': self.actor = CatActor(n_states, n_actions, hidden_size) else: self.actor = NumActor(n_states, n_actions, hidden_size) self.actor_opt = torch.optim.Adam(self.actor.parameters(), lr=alr) self.critic = V_Critic(n_states, 1, hidden_size) self.critic_opt = torch.optim.Adam(self.critic.parameters(), lr=clr) self.MseLoss = torch.nn.MSELoss() self.step = 0
def __init__(self, path, date_format, start_date, positive_window_size, #manufacturer, \ disk_model, columns, features, label, forget_type, bl_delay=False, \ dropna=False, negative_window_size=6, validation_window=6, \ bl_regression=False, label_days=None, bl_transfer=False, bl_ssd=False): super().__init__() self.memory = Memory(path, start_date, positive_window_size, #manufacturer,\ disk_model, columns, features, label, forget_type, dropna, bl_delay, \ negative_window_size, bl_regression, label_days, bl_transfer, date_format, bl_ssd) if not bl_transfer: self.memory.buffering() self.data = self.memory.ret_df.drop(['model', 'date'], axis=1) else: self.data = self.memory.ret_df.drop(['model', 'date'], axis=1) self.data = self.data.reset_index(drop=True) self.class_name = label[0] self.num_classes = 2 self.bl_delay = bl_delay self.validation_window = validation_window
def __init__(self, paramsManager): self.paramsManager = paramsManager self.memory = Memory( self.paramsManager.get_params()["agent"]["GOOD_MEMORIES_SIZE"], self.paramsManager.get_params()["agent"]["BAD_MEMORIES_SIZE"], self.paramsManager.get_params()["agent"]["MINI_BATCH_SIZE"], self.paramsManager.get_params()["environment"] ["FRAME_PROCESSED_WIDTH"], self.paramsManager.get_params()["environment"] ["FRAME_PROCESSED_HEIGHT"], self.paramsManager.get_params()["environment"] ["NUMBER_OF_FRAMES_TO_STACK_ON_STATE"]) print("[i] Creating main convolutional neural network") self.main_cnn = CNN() print("[i] Creating target convolutional neural network") self.target_cnn = copy.deepcopy(self.main_cnn) print("[!] Creating the agent") self.main_cnn.cuda() self.target_cnn.cuda() self.agent = Agent( self.main_cnn, self.target_cnn, self.paramsManager.get_params()["agent"]["EPSILON_MAX"], self.paramsManager.get_params()["agent"] ["NUMBER_OF_FRAMES_WITH_CONSTANT_EPSILON"], self.paramsManager.get_params()["agent"]["FIRST_EPSILON_DECAY"], self.paramsManager.get_params()["agent"] ["FRAMES_TO_FIRST_EPSILON_DECAY"], self.paramsManager.get_params()["agent"]["FINAL_EPSILON_VALUE"], self.paramsManager.get_params()["agent"] ["FRAMES_TO_FINAL_EPSILON"], self.paramsManager.get_params()["agent"] ["EXPLORATION_PROBABILITY_DURING_EVALUATION"], self.paramsManager.get_params()["agent"]["LEARNING_RATE"]) self.breakout_wrapper = BreakoutWrapper( self.paramsManager.get_params()["environment"]["NAME"], self.paramsManager.get_params()["agent"]["NO_OP_STEPS"], self.paramsManager.get_params()["environment"] ["NUMBER_OF_FRAMES_TO_STACK_ON_STATE"], self.paramsManager.get_params()["environment"] ["FRAME_PROCESSED_WIDTH"], self.paramsManager.get_params()["environment"] ["FRAME_PROCESSED_HEIGHT"], self.paramsManager.get_params()["environment"]["RENDER"])
def perform_rollout(self, theta, inner=False): memory = Memory(self.hp) (s1, s2), _ = self.env.reset() for t in range(self.hp.len_rollout): a1, lp1 = self.act(s1, self.theta) a2, lp2 = self.act_opp(s2, theta) if self.id > 0: (s2, s1), (r2, r1), _, _ = self.env.step((a2, a1)) else: (s1, s2), (r1, r2), _, _ = self.env.step((a1, a2)) r1 = torch.Tensor(r1) r2 = torch.Tensor(r2) if inner: memory.add(lp2, lp1, r2) else: memory.add(lp1, lp2, r1) return memory
def Game(max_ep_len=1000, num_frames=4): global exit_game global actions env = gym.make('CarRacing-v0') state_dim = env.observation_space.shape action_dim = env.action_space.shape print(f"State: {state_dim}") print(f"Action: {action_dim}") # set interrupts env.reset() env.viewer.window.on_key_press = key_press env.viewer.window.on_key_release = key_release # make global actions array actions = np.zeros(4, dtype=np.float32) # mem memory = Memory() memory.create(state_dim, action_dim) # logger ep_ret_log = [] # init environment obs, ep_ret, ep_len, epoch = env.reset(), 0, 0, 0 obs = np.expand_dims(obs, axis=0) state_stack = np.repeat(obs, num_frames, axis=0) print(state_stack.shape) print(state_stack.dtype) # main loop while exit_game == False: # render window env.render() # take action obs2, r, d, _ = env.step(actions[:3]) obs2 = np.expand_dims(obs2, axis=0) state_stack = np.append(state_stack[1:], obs2, axis=0) # statistics ep_ret += r ep_len += 1 # Ignore the 'done' signal d = False if ep_len == max_ep_len else d # store in memory memory.add(state_stack, np.array(actions[:3]), r, d) # End of episode if d or (ep_len == max_ep_len): print(f"Epoch: {epoch}, EpRet: {ep_ret}, EpLen: {ep_len}, ReplayBuff: {len(memory)}") # if exists statistical data if len(ep_ret_log) > 0: log = np.array(ep_ret_log) print("AvgEpRet:", log.mean()) print("StdEpRet:", log.std()) print("MaxEpRet:", log.max()) print("MinEpRet:", log.min()) print() ep_ret_log.append(ep_ret) obs, ep_ret, ep_len = env.reset(), 0, 0 obs = np.expand_dims(obs, axis=0) state_stack = np.repeat(obs, num_frames, axis=0) epoch += 1 print('\n') # save the dataset memory.save()
def main(): # initialize the game env = gym.make('Pendulum-v0').unwrapped print env.observation_space print env.observation_space.high print env.observation_space.low print env.action_space # import hyper parameters args = init_hyper_para() # random initialize critic network state_dim = env.reset().shape[0] action_dim = env.action_space.shape[0] # if we have the saved model, load it if os.path.exists( '/home/likang/PycharmProjects/myddpg/bin/Models/critic.ckpt'): critic_net = torch.load( '/home/likang/PycharmProjects/myddpg/bin/Models/critic.ckpt') else: # initialize the model critic_net = net.CriticNetwork( state_dim=state_dim, action_dim=action_dim).to( device) # need to init paras according to the gym game # random initialize actor network(also called policy network) if os.path.exists( '/home/likang/PycharmProjects/myddpg/bin/Models/actor.ckpt'): actor_net = torch.load( '/home/likang/PycharmProjects/myddpg/bin/Models/actor.ckpt') else: actor_net = net.ActorNetwork(state_dim=state_dim, action_dim=action_dim).to(device) # initialize optimizer_critic = opt.Adam(critic_net.parameters(), lr=0.001) optimizer_actor = opt.Adam(actor_net.parameters(), lr=0.001) # initialize target critic network which is the same of critic network target_critic_net = copy.deepcopy(critic_net) # initialize target actor network which is the same of actor network target_actor_net = copy.deepcopy(actor_net) # init the memory buffer memory = Memory(args.capacity) # initialize a random process N for action exploration ounoise = OUNoise(env.action_space.shape[0]) # init random process # enter circle of training process for ep in range(args.num_ep): print(["ep: ", ep]) # reset random process ounoise.scale = (args.noise_scale - args.final_noise_scale) * max( 0, args.exploration_end - ep) / args.exploration_end + args.final_noise_scale ounoise.reset() # initialize a state s1 state = env.reset() # 这里把state初始化成二维的tensor state = torch.tensor([state], dtype=torch.float32).to(device) for t in range(MAX_STEP): print(['time step: ', t]) # select a action according to actor network(also called policy network) action = actor_net.select_action(state, ounoise) # execute the action and get a new state s_i+i # get a reward from the environment next_state, reward, done, _ = env.step([action.item()]) # store the transition {s_i, a_i, r_i, s_i+1} into memory next_state = torch.tensor([next_state], device=device, dtype=torch.float32) reward = torch.tensor([[reward]], device=device, dtype=torch.float32) memory.push(state, action, reward, next_state) state = next_state # print([state, action, reward, next_state]) del action, reward, next_state # get a batch_size transitions. # (s_i, a_i, r_i, s_{i+1}) in Algorithm1 of DDPG transitions = memory.sample(args.batch_size) s1 = torch.cat([tran.state for tran in transitions]) s2 = torch.cat([tran.next_state for tran in transitions]) r1 = torch.cat([tran.reward for tran in transitions]) a1 = torch.cat([tran.action for tran in transitions]) update_critic_net(s1, s2, r1, a1, target_actor_net, target_critic_net, critic_net, optimizer_critic, args) # update actor policy network update_actor_net(s1, actor_net, critic_net, optimizer_actor) # update target critic network # theta^{Q'}, see algorithm1 of DDPG for target_param, source_param in zip( target_critic_net.parameters(), critic_net.parameters()): target_param.data.copy_(args.tau * source_param + (1 - args.tau) * target_param) # update target actor network # theta^{mu'}, see algorithm1 of DDPG for target_param, source_param in zip( target_actor_net.parameters(), actor_net.parameters()): target_param.data.copy_(args.tau * source_param + (1 - args.tau) * target_param) # show image plt.imshow(env.render('rgb_array')) time.sleep(0.001) # finish if done: break del transitions gc.collect() if ep % 10 == 0: # save model torch.save(critic_net, './Models/' + 'critic.ckpt') torch.save(actor_net, './Models/' + 'actor.ckpt')
def __init__(self): super().__init__() self.memory = Memory(1000)
def setup_memory(self) -> None: columns = [ "states", "next_states", "actions", "log_probs", "rewards", "done" ] self.episode_memory = Memory(columns) self.epoch_memory = Memory(columns)
def train(): numAgent = 10 # multiple agents are running synchronously. # each agent has a different type with different properties. # Only one network is created, different agent gets their # own behavior according to the embedding input. numGame = 20 # multiple games running simultaneously. print('agent count:', numAgent) print('Env num:', numGame) env = {} for game in range(numGame): env[game] = miniDotaEnv(args, numAgent) # initialize the neural networks. # use a single network to share the knowledge. net = ac(args) if not args.cpuSimulation: net = net.to(device) if args.load_model is not None: saved_ckpt_path = os.path.join(os.getcwd(), 'save_model', str(args.load_model)) ckpt = torch.load(saved_ckpt_path) net.load_state_dict(ckpt['net']) observations, lastDone = {}, {} for game in range(numGame): observations[game] = env[game].reset(0)[ 'observations'] # get initial state. lastDone[game] = [ False ] * 10 # to record whether game is done at the previous step. optimizer = optim.Adam(net.parameters(), lr=args.lr) for iteration in range(args.max_iter): # playing-training iteration. start = time.time() print() print('Start iteration %d ..' % iteration) if args.cpuSimulation: net = net.cpu() net.eval() # switch to evaluation mode. memory = [] for i in range(numGame): memory.append([Memory() for j in range(numAgent)]) # memory is cleared at every iter so only the current iteration's samples are used in training. # the separation of memory according to game is necessary as they # need to be processed separate for each game. steps = 0 teamscore = 0 # only for game 0. record = [] # record the states for visualization. gameEnd = np.zeros(numGame).astype(bool) while steps <= args.time_horizon: # loop for one game. if np.all(gameEnd): break steps += 1 stateList = [] for game in range(numGame): for agent in range(numAgent): stateList.append( np.expand_dims(observations[game][agent], axis=0)) stateCombined = np.concatenate(stateList, axis=0) # concatenate the states of all games and process them by the network together. with torch.no_grad(): actionDistr = net(to_tensor(stateCombined, args.cpuSimulation)) actions = get_action(actionDistr) for game in range(numGame): if not gameEnd[game]: # the following random action cannot work, because random action has too small prob density value, # leading to strange bugs. # sample = random.random() # if sample > args.randomActionRatio * (1 - min(1, iteration/1000) ): # thisGameAction = actions[10*game:10*(game+1), :] # contain actions from all agents. # check(thisGameAction) # else: # actionmove = np.random.randint(0, 3, size=(10,3)) # target = np.random.randint(0, 12, size=(10,1)) # thisGameAction = np.concatenate([actionmove, target], axis=1) thisGameAction = actions[10 * game:10 * ( game + 1 ), :] # select the actions from all agents of this env. envInfo = env[game].step( thisGameAction ) # environment runs one step given the action. nextObs = envInfo['observations'] # get the next state. if game == 0: record.append( np.concatenate([ env[game].getState(), actions[0:10, :].reshape(-1) ])) rewards = envInfo['rewards'] dones = envInfo['local_done'] # masks = list(~dones) # cut the return calculation at the done point. masks = [ True ] * numAgent # no need to mask out the last state-action pair, # because the last reward is useful to us. for i in range(numAgent): if not lastDone[game][i]: memory[game][i].push(observations[game][i], thisGameAction[i], rewards[i], masks[i]) lastDone[game] = dones if game == 0: teamscore += sum( [rewards[x] for x in env[game].getTeam0()]) observations[game] = nextObs gameEnd[game] = np.all(dones) if gameEnd[game]: if game == 0: print('Game 0 score: %f' % teamscore) # recordMat = np.stack(record)# stack will expand the dimension before concatenate. # draw(recordMat, iteration, env[game].getUnitRange(), 10) observations[game] = env[game].reset(iteration + 1)['observations'] lastDone[game] = [False] * 10 simEnd = time.time() print('Simulation time: %.f' % (simEnd - start)) net.train() # switch to training mode. net = net.cuda() sts, ats, returns, advants, old_policy, old_value = [], [], [], [], [], [] for game in range(numGame): for i in range(numAgent): batch = memory[game][i].sample() st, at, rt, adv, old_p, old_v = process_memory( net, batch, args) sts.append(st) ats.append(at) returns.append(rt) advants.append(adv) old_policy.append(old_p) old_value.append(old_v) sts = torch.cat(sts) ats = torch.cat(ats) returns = torch.cat(returns) advants = torch.cat(advants) old_policy = torch.cat(old_policy) old_value = torch.cat(old_value) train_model(net, optimizer, sts, ats, returns, advants, old_policy, old_value, args) # training is based on the state-action pairs from all games of the current iteration. trainEnd = time.time() print('Training time: %.f' % (trainEnd - simEnd)) if iteration % 10 == 0: model_path = os.path.join(os.getcwd(), 'save_model') if not os.path.isdir(model_path): os.makedirs(model_path) ckpt_path = os.path.join(model_path, 'ckpt_%.3f.pth.tar' % teamscore) save_checkpoint( { 'net': net.state_dict(), 'args': args, 'score': teamscore }, filename=ckpt_path)
critic_layer_sizes, grayscale=grayscale).to(device) # Create AE Hashing model and optimizers ae_hash = AEHash(len_AE_hashcode, 4 if stacked else 1, noise_scale, saturating_weight, device=device).to(device) ae_hash_optim = optim.Adam(ae_hash.parameters()) # Create SimHash sim_hash = SimHash(len_AE_hashcode, len_SimHash_hashcode) # Set up memory memory = Memory(capacity, GAMMA, LAMBDA, 'cpu') # Put memory on cpu to save space # Set up pixel observation preprocessing transform = Compose([ ToPILImage(), Grayscale(num_output_channels=1), # Turn frame into grayscale Resize((52, 52)), ToTensor() ]) ################################################################### # Start training # Dictionary for extra training information to save to checkpoints training_info = { "epoch mean durations": [],
print("Loaded OK ex. Zfilter N {}".format(running_state.rs.n)) states = running_state(env_info.vector_observations) actor_optim = optim.Adam(actor.parameters(), lr=args.actor_lr) critic_optim = optim.Adam(critic.parameters(), lr=args.critic_lr, weight_decay=args.l2_rate) scores = [] score_avg = 0 for iter in range(args.max_iter): actor.eval(), critic.eval() memory = [Memory() for _ in range(num_agent)] steps = 0 score = 0 while steps < args.time_horizon: steps += 1 mu, std, _ = actor(to_tensor(states)) actions = get_action(mu, std) env_info = env.step(actions)[default_brain] next_states = running_state(env_info.vector_observations) rewards = env_info.rewards dones = env_info.local_done masks = list(~(np.array(dones)))
axis=0), mask) if done: print(step_no, 'epsiode reward- ' + str(eps_reward)) eps_reward = 0 state = env.reset() state_list = [] for i in range(args.past_frames - 1): state_list.append(initial[0]) state_list.append(state[0]) if len(memory) < args.batch_size: batch = memory.sample() else: batch = memory.sample(size=args.batch_size) update_dqn(model, batch, args, criterion, optimizer, device) step_no += 1 inp_channels = 4 env = gym.make(args.env_name) env = AtariRescale105x80(env) num_actions = env.action_space.n dqn = DQN(inp_channels, num_actions) memory = Memory(limit=args.replay_size) device = torch.device('cuda') dqn.to(device) train_model(dqn, env, memory, args, device)
def main(cfg): random.seed(cfg.exp.seed) np.random.seed(cfg.exp.seed) torch.manual_seed(cfg.exp.seed) torch.backends.cudnn.deterministic = cfg.exp.torch_deterministic # so that the environment automatically resets env = SyncVectorEnv([ lambda: RecordEpisodeStatistics(gym.make('CartPole-v1')) ]) actor, critic = Actor(), Critic() actor_optim = Adam(actor.parameters(), eps=1e-5, lr=cfg.params.actor_lr) critic_optim = Adam(critic.parameters(), eps=1e-5, lr=cfg.params.critic_lr) memory = Memory(mini_batch_size=cfg.params.mini_batch_size, batch_size=cfg.params.batch_size) obs = env.reset() global_rewards = [] NUM_UPDATES = (cfg.params.total_timesteps // cfg.params.batch_size) * cfg.params.epochs cur_timestep = 0 def calc_factor(cur_timestep: int) -> float: """Calculates the factor to be multiplied with the learning rate to update it.""" update_number = cur_timestep // cfg.params.batch_size total_updates = cfg.params.total_timesteps // cfg.params.batch_size fraction = 1.0 - update_number / total_updates return fraction actor_scheduler = LambdaLR(actor_optim, lr_lambda=calc_factor, verbose=True) critic_scheduler = LambdaLR(critic_optim, lr_lambda=calc_factor, verbose=True) while cur_timestep < cfg.params.total_timesteps: # keep playing the game obs = torch.as_tensor(obs, dtype=torch.float32) with torch.no_grad(): dist = actor(obs) action = dist.sample() log_prob = dist.log_prob(action) value = critic(obs) action = action.cpu().numpy() value = value.cpu().numpy() log_prob = log_prob.cpu().numpy() obs_, reward, done, info = env.step(action) if done[0]: tqdm.write(f'Reward: {info[0]["episode"]["r"]}, Avg Reward: {np.mean(global_rewards[-10:]):.3f}') global_rewards.append(info[0]['episode']['r']) wandb.log({'Avg_Reward': np.mean(global_rewards[-10:]), 'Reward': info[0]['episode']['r']}) memory.remember(obs.squeeze(0).cpu().numpy(), action.item(), log_prob.item(), reward.item(), done.item(), value.item()) obs = obs_ cur_timestep += 1 # if the current timestep is a multiple of the batch size, then we need to update the model if cur_timestep % cfg.params.batch_size == 0: for epoch in tqdm(range(cfg.params.epochs), desc=f'Num updates: {cfg.params.epochs * (cur_timestep // cfg.params.batch_size)} / {NUM_UPDATES}'): # sample a batch from memory of experiences old_states, old_actions, old_log_probs, old_rewards, old_dones, old_values, batch_indices = memory.sample() old_log_probs = torch.tensor(old_log_probs, dtype=torch.float32) old_actions = torch.tensor(old_actions, dtype=torch.float32) advantage = calculate_advantage(old_rewards, old_values, old_dones, gae_gamma=cfg.params.gae_gamma, gae_lambda=cfg.params.gae_lambda) advantage = torch.tensor(advantage, dtype=torch.float32) old_rewards = torch.tensor(old_rewards, dtype=torch.float32) old_values = torch.tensor(old_values, dtype=torch.float32) # for each mini batch from batch, calculate advantage using GAE for mini_batch_index in batch_indices: # remember: Normalization of advantage is done on mini batch, not the entire batch advantage[mini_batch_index] = (advantage[mini_batch_index] - advantage[mini_batch_index].mean()) / (advantage[mini_batch_index].std() + 1e-8) dist = actor(torch.tensor(old_states[mini_batch_index], dtype=torch.float32).unsqueeze(0)) # actions = dist.sample() log_probs = dist.log_prob(old_actions[mini_batch_index]).squeeze(0) entropy = dist.entropy().squeeze(0) log_ratio = log_probs - old_log_probs[mini_batch_index] ratio = torch.exp(log_ratio) with torch.no_grad(): # approx_kl = ((ratio-1)-log_ratio).mean() approx_kl = ((old_log_probs[mini_batch_index] - log_probs)**2).mean() wandb.log({'Approx_KL': approx_kl}) actor_loss = -torch.min( ratio * advantage[mini_batch_index], torch.clamp(ratio, 1 - cfg.params.actor_loss_clip, 1 + cfg.params.actor_loss_clip) * advantage[mini_batch_index] ).mean() values = critic(torch.tensor(old_states[mini_batch_index], dtype=torch.float32).unsqueeze(0)).squeeze(-1) returns = old_values[mini_batch_index] + advantage[mini_batch_index] critic_loss = torch.max( (values - returns)**2, (old_values[mini_batch_index] + torch.clamp( values - old_values[mini_batch_index], -cfg.params.critic_loss_clip, cfg.params.critic_loss_clip ) - returns )**2 ).mean() # critic_loss = F.mse_loss(values, returns) wandb.log({'Actor_Loss': actor_loss.item(), 'Critic_Loss': critic_loss.item(), 'Entropy': entropy.mean().item()}) loss = actor_loss + 0.25 * critic_loss - 0.01 * entropy.mean() actor_optim.zero_grad() critic_optim.zero_grad() loss.backward() nn.utils.clip_grad_norm_(actor.parameters(), cfg.params.max_grad_norm) nn.utils.clip_grad_norm_(critic.parameters(), cfg.params.max_grad_norm) actor_optim.step() critic_optim.step() memory.reset() actor_scheduler.step(cur_timestep) critic_scheduler.step(cur_timestep) y_pred, y_true = old_values.cpu().numpy(), (old_values + advantage).cpu().numpy() var_y = np.var(y_true) explained_var = np.nan if var_y == 0 else 1 - np.var(y_true - y_pred) / var_y wandb.log({'Explained_Var': explained_var}) if cfg.exp.save_weights: torch.save(actor.state_dict(), Path(f'{hydra.utils.get_original_cwd()}/{cfg.exp.model_dir}/actor.pth')) torch.save(critic.state_dict(), Path(f'{hydra.utils.get_original_cwd()}/{cfg.exp.model_dir}/critic.pth'))
} for i in range(n_actor): agent.store_policy('Pendulum-v0', score=fs[i], index=i) n += 1 # printing iteration resume if debug: prPurple('Iteration#{}: Total steps:{} \n'.format(n, total_steps)) if __name__ == "__main__": # The environment env = gym.make("Pendulum-v0") state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] max_action = int(env.action_space.high[0]) # replay buffer memory = Memory(100000, state_dim, action_dim) # agent agent = DTD3(state_dim, action_dim, max_action, memory, n_actor=5) print("starting") train(agent, n_episodes=1000, max_steps=1000000, debug=True, n_eval=100, n_actor=5)
args = parser.parse_args() args.output = get_output_folder(args.output, args.env) args.use_cuda = USE_CUDA with open(args.output + "/parameters.txt", 'w') as file: for key, value in vars(args).items(): file.write("{} = {}\n".format(key, value)) # Environment env = gym.make(args.env) state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] max_action = int(env.action_space.high[0]) # Memory memory = Memory(args.mem_size, state_dim, action_dim, args) # Algorithm drla = NASTD3(state_dim, action_dim, max_action, args) # Action noise a_noise = GaussianNoise(action_dim, sigma=args.gauss_sigma) # Logger fields = ["eval_score", "critic_loss", "actor_loss", "total_steps"] logger = Logger(args.output, fields) # Train ite = 0 K = 5000 total_steps = 0
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("Current usable device is: ", device) # Create the model. Two value net policy_net = PolicyNet(layer_sizes).to(device) # Policy network value_net_ex = ValueNet(input_size).to( device) # Value network for extrinsic reward value_net_in = ValueNet(input_size + 1 + output_size).to( device) # One additional input unit to indicate trajectory number # Set up optimizer valuenet_in_optimizer = optim.Adam(value_net_in.parameters()) valuenet_ex_optimizer = optim.Adam(value_net_ex.parameters()) # Set up memory memory = Memory(capacity, GAMMA, LAMBDA, device=device) # Define observation normalization function. Normalize state vector values to range [-1., 1.] def state_nomalize(s): # Obtain environment observation space limit high = env.observation_space.high low = env.observation_space.low return ((s - low) / (high - low)) * 2 - 1 # Create Hashing function simhash = SimHash(input_size, len_hashcode, preprocessor=state_nomalize if use_preprocessor else None)
def marl_test(config): experiment_name = config.setdefault("experiment_name", "") time_slots = config.setdefault("time_slots", 10000) simulations = config.setdefault("simulations", 3) memory_size = config.setdefault("memory_size", 1200) pretrain_length = config.setdefault("pretrain_length", 6) step_size = config.setdefault("step_size", 5) save_freq = config.setdefault("save_freq", 1000) save_results = config.setdefault("save_results", True) save_model = config.setdefault("save_model", False) load_model = config.setdefault("load_model", False) load_slot = config.setdefault("load_slot", 4999) training = config.setdefault("training", False) episode_interval = config.setdefault("episode_interval", 25) explore_step = config.setdefault("explore", 2000) greedy_step = config.setdefault("greedy", 20000) training_stop = config.setdefault("training_stop", 20000) # Stop the training after these time step. train_after_episode = config.setdefault("train_after_episode", False) # Train after each episode in stead of training after each time slot. global_reward_avg = config.setdefault("global_reward_avg", False) # Train after each episode in stead of training after each time slot. save_positions = config.setdefault("save_positions", False) # Train after each episode in stead of training after each time slot. enable_channel = config.setdefault("enable_channel", False) # Train after each episode in stead of training after each time slot. batch_size = config["RLAgent"]["batch_size"] ia_penalty_enable = config.setdefault("ia_penalty_enable", False) ia_averaging = config.setdefault("ia_averaging", False) for simulation in range(simulations): print("-=-=-=-=-=-=-=-=-=-=-= experiment_name: " + experiment_name + " SIMULATION " + str(simulation + 1) + " =-=-=-=-=-=-=-=-=-=-=-") # Initialize the env. env = TestEnv(**config["EnvironmentTest"]) if ia_penalty_enable: ia_penalty_threshold = config.setdefault("ia_penalty_threshold", 5) ia_penalty_value = config.setdefault("ia_penalty_value", -10) ia_penalty_counter = {} previous_actions = {} # store the previous taken action by the UE. num_users = env.get_total_users() for user in range(num_users): ia_penalty_counter[user] = 0 previous_actions[user] = -1 # Initialize the agen mainDRQN = DRQN(env, name=experiment_name, total_episodes=time_slots/episode_interval, **config["RLAgent"]) #mainDRQN = DeepRecurrentQNetwork(env=env, name=experiment_name, **config["RLAgent"]) if load_model: print("Load model DRQN time step " + str(load_slot)) save_dir = "save_model/" + "test/" mainDRQN.load_model(save_dir, load_slot) # this is experience replay buffer(deque) from which each batch will be sampled and fed to the neural network for training memory = Memory(max_size=memory_size) log_reward_slot = [] log_actions_slot = [] log_ia_slot = [] sum_ia_prev = 0 log_x_positions = [] start_time = time.time() episode = 0 # Used to update the greediness of the algorithm # cumulative reward cum_r = [0] cum_r_slots = [0] # cumulative collision cum_collision = [0] cum_collision_slots = [0] # this is our input buffer which will be used for predicting next Q-values history_input = deque(maxlen=step_size) # env.network.reset_ia() # to sample random actions for each user action = env.sample() #obs = env.step(action) obs, rews = env.my_step(action, 0) rews = list(rews) state = env.obtain_state(obs, action, rews) # reward = [i[1] for i in obs[:num_users]] num_users = env.get_total_users() num_channels = env.get_action_space() ############################################## for ii in range(pretrain_length*step_size*5): action = env.sample() if enable_channel: obs, reward = env.my_step_ch(action, 0) # obs is a list of tuple with [(ACK,REW) for each user ,(CHANNEL_RESIDUAL_CAPACITY_VECTOR)] else: #obs, reward = env.my_step( # action, 0) # obs is a list of tuple with [(ACK,REW) for each user ,(CHANNEL_RESIDUAL_CAPACITY_VECTOR)] obs, reward = env.my_step_design(action, 0) # obs is a list of tuple with [[(ACK,REW) for each user] ,CHANNEL_RESIDUAL_CAPACITY_VECTOR] next_state = env.obtain_state(obs, action, rews) #next_state = env.state_generator(action, obs) memory.add((state, action, rews, next_state)) state = next_state history_input.append(state) ############################################## # TODO: now load the positions env.load_saved_positions() for time_step in range(time_slots): #initializing action vector action = np.zeros([num_users], dtype=np.int32) #converting input historskyy into numpy array # TODO: enable below for lstm state_vector = np.array(history_input) # LSTM # state_vector = state # DQN for each_user in range(num_users): #action[each_user] = mainDRQN.infer_action(each_user, state_vector=state_vector, time_slot=time_step) if time_step < explore_step and not load_model: # and 0: action[each_user] = mainDRQN.infer_action(each_user, state_vector=state_vector, episode=episode, policy="explore") elif time_step < greedy_step and not load_model: # and 0: action[each_user] = mainDRQN.infer_action(each_user, state_vector=state_vector, episode=episode) else: action[each_user] = mainDRQN.infer_action(each_user, state_vector=state_vector, episode=episode, policy="greedy") # taking action as predicted from the q values and receiving the observation from the envionment # obs = env.step(action) # obs is a list of tuple with [(ACK,REW) for each user ,(CHANNEL_RESIDUAL_CAPACITY_VECTOR)] if save_positions: user_pos = env.get_x_pos() log_x_positions.append(user_pos) if enable_channel: obs, reward = env.my_step_ch(action, time_step) # obs is a list of tuple with [(ACK,REW) for each user ,(CHANNEL_RESIDUAL_CAPACITY_VECTOR)] else: obs, reward = env.my_step(action, time_step) # obs is a list of tuple with [(ACK,REW) for each user ,(CHANNEL_RESIDUAL_CAPACITY_VECTOR)] #obs, reward = env.my_step_design(action, time_step) # TODO: update the env topology after each step. log_actions_slot.append(action) ia = env.network.get_information_age(time_step) ia_sum = calculate_ia_penalty(ia) log_ia_slot.append(ia) if ia_averaging: # ia based penalty to the reward ia_penalty = 0 if ia_sum > sum_ia_prev: ia_penalty = -1 elif ia_sum < sum_ia_prev: ia_penalty = 1 sum_ia_prev = ia_sum # Generate next state from action and observation # next_state = env.state_generator(action, obs) used for DQN next_state = env.obtain_state(obs, action, reward, episode, mainDRQN.get_eps()) # print (next_state) # reward for all users given by environment #reward = [i[1] for i in obs[:num_users]] # calculating sum of rewards sum_r = np.sum(reward) #calculating cumulative reward cum_r.append(cum_r[-1] + sum_r) cum_r_slots.append(cum_r_slots[-1] + sum_r) #If NUM_CHANNELS = 2 , total possible reward = 2 , therefore collision = (2 - sum_r) or (NUM_CHANNELS - sum_r) collision = num_channels - sum_r #calculating cumulative collision cum_collision.append(cum_collision[-1] + collision) cum_collision_slots.append(cum_collision_slots[-1] + collision) ############################# # for co-operative policy we will give reward-sum to each user who have contributed # to play co-operatively and rest 0 # NOTE: I think, I do not need that part since I already use positive and negative reward. for i in range(len(reward)): # for each user we have this. #if reward[i] > 0: if ia_averaging: # add penalty based on the direction of the Information age. reward[i] += ia_penalty if ia_penalty_enable: if reward[i] < 1 and action[i] == previous_actions[i]: ia_penalty_counter[i] += 1 else: ia_penalty_counter[i] = 0 if ia_penalty_counter[i] > ia_penalty_threshold: reward[i] = ia_penalty_value previous_actions[i] = action[i] if global_reward_avg: reward[i] = reward[i] + sum_r/len(reward) # Add the average total reward to each UE. ############################# #reward = reward*2 # Add the average total reward to each UE. log_reward_slot.append(sum_r) # print (reward) # print("EPOCH " + str(time_step)) # add new experiences into the memory buffer as (state, action , reward , next_state) for training memory.add((state, action, reward, next_state)) state = next_state #add new experience to generate input-history sequence for next state history_input.append(state) # Start training. if not train_after_episode: if time_step < training_stop and training: #and not load_model: mainDRQN.train(memory, time_step) if time_step%(episode_interval) == episode_interval-1: print("Time step " + str(time_step) + " epsilon " + str(mainDRQN.get_eps()) + " cum Collison " + str(cum_collision[episode_interval]) + " sum reward " + str(cum_r[episode_interval]) + " total time " + str(time.time()-start_time) ) cum_r = [0] cum_collision = [0] episode += 1 # Updates the velocity of the vehicles if activated env.update_velocity() # ia = env.network.get_information_age(time_step) if train_after_episode and time_step > (batch_size+10) and training: mainDRQN.train(memory, time_step) if time_step%save_freq == save_freq-1: # Save the collisions if save_results: print("save results for timestep ", time_step + 1) save_dir = "save_results/" + "test/" save_dir = save_dir + experiment_name if not os.path.isdir(save_dir): os.makedirs(save_dir) # filename = save_dir + "/collisions" + "_" + str(time_step) +"_sim"+str(simulation) # np.save(filename, np.asarray(cum_collision_slots)) filename = save_dir + "/rewards" + "_sim"+str(simulation) np.save(filename, np.asarray(log_reward_slot)) filename = save_dir + "/actions" + "_sim"+str(simulation) np.save(filename, np.asarray(log_actions_slot)) # filename = save_dir + "/time_step" + "_" + str(time_step)+"_sim"+str(simulation) # np.save(filename, np.asarray(str(time.time()-start_time))) filename = save_dir + "/positions" + "_sim"+str(simulation) np.save(filename, np.asarray(log_x_positions)) #filename = save_dir + "/ia" + "_sim"+str(simulation) #np.save(filename, np.asarray(log_ia_slot)) #"_" + str(time_step)+ if save_model: print("save model for timestep ", time_step + 1) save_dir = "save_model/" + "test/" #save_dir = save_dir mainDRQN.save_model(save_dir, time_step,simulation)