class DDPG: """docstring for DDPG""" def __init__(self, state_dim, action_dim, env): self.name = 'DDPG' # name for uploading results self.time_step = 0 self.state_dim = state_dim self.action_dim = action_dim self.environment = env self.sess = tf.InteractiveSession() self.actor_network = ActorNetwork(self.sess, self.state_dim, self.action_dim) self.critic_network = CriticNetwork(self.sess, self.state_dim, self.action_dim) # initialize replay buffer self.replay_buffer = ReplayBuffer(REPLAY_BUFFER_SIZE) # Initialize a random process the Ornstein-Uhlenbeck process for action exploration self.exploration_noise = OUNoise(self.action_dim) def train(self): minibatch = self.replay_buffer.get_batch(BATCH_SIZE) state_batch = np.asarray([data[0] for data in minibatch]) action_batch = np.asarray([data[1] for data in minibatch]) reward_batch = np.asarray([data[2] for data in minibatch]) next_state_batch = np.asarray([data[3] for data in minibatch]) done_batch = np.asarray([data[4] for data in minibatch]) # for action_dim = 1 action_batch = np.resize(action_batch, [BATCH_SIZE, self.action_dim]) # Calculate y_batch next_action_batch = self.actor_network.target_actions(next_state_batch) q_value_batch = self.critic_network.target_q(next_state_batch, next_action_batch) y_batch = [] for i in range(len(minibatch)): if done_batch[i]: y_batch.append(reward_batch[i]) else: y_batch.append(reward_batch[i] + GAMMA * q_value_batch[i]) y_batch = np.resize(y_batch, [BATCH_SIZE, 1]) # Update critic by minimizing the loss L self.critic_network.train(y_batch, state_batch, action_batch) # Update the actor policy using the sampled gradient: action_batch_for_gradients = self.actor_network.actions(state_batch) q_gradient_batch = self.critic_network.gradients( state_batch, action_batch_for_gradients) self.actor_network.train(q_gradient_batch, state_batch) # Update the target networks self.actor_network.update_target() self.critic_network.update_target() def noise_action(self, state): # Select action a_t according to the current policy and exploration noise action = self.actor_network.action(state) return action + self.exploration_noise.noise() def action(self, state): action = self.actor_network.action(state) return action def perceive(self, state, action, reward, next_state, done): # Store transition (s_t,a_t,r_t,s_{t+1}) in replay buffer self.replay_buffer.add(state, action, reward, next_state, done) if self.replay_buffer.count() == REPLAY_START_SIZE: print('\n---------------Start training---------------') # Store transitions to replay start size then start training if self.replay_buffer.count() > REPLAY_START_SIZE: self.time_step += 1 self.train() if self.time_step % 10000 == 0 and self.time_step > 0: self.actor_network.save_network(self.time_step) self.critic_network.save_network(self.time_step) # Re-iniitialize the random process when an episode ends if done: self.exploration_noise.reset() return self.time_step
class DDPG: """docstring for DDPG""" def __init__(self, state_dim, action_dim): """name for uploading resuults""" self.name = 'DDPG' self.time_step = 0 # self.atten_rate = 1 """Randomly initialize actor network and critic network""" """and both their target networks""" self.state_dim = state_dim self.action_dim = action_dim self.sess = tf.InteractiveSession() self.actor_network = ActorNetwork(self.sess, self.state_dim, self.action_dim) self.critic_network = CriticNetwork(self.sess, self.state_dim, self.action_dim) """initialize replay buffer""" self.replay_buffer = ReplayBuffer(REPLAY_BUFFER_SIZE) """Initialize a random process the Ornstein-Uhlenbeck process for action exploration""" self.exploration_noise = OUNoise(self.action_dim) """Initialize a Treading""" self.threading = threading.Thread(target=self.train, name='LoopThread--DDPG') def train(self): # if self.time_step ==0: # print("Begins Training!!!") #print("Training Begins") self.time_step += 1 """Sample a random minibatch of N transitions from replay buffer""" """take out BATCH_SIZE sets of data""" minibatch = self.replay_buffer.get_batch(BATCH_SIZE) state_batch = np.asarray([data[0] for data in minibatch]) action_batch = np.asarray([data[1] for data in minibatch]) reward_batch = np.asarray([data[2] for data in minibatch]) next_state_batch = np.asarray([data[3] for data in minibatch]) done_batch = np.asarray([data[4] for data in minibatch]) """resize the action_batch shape to [BATCH_SIZE, self.action_dim]""" action_batch = np.resize(action_batch, [BATCH_SIZE, self.action_dim]) """Calculate y_batch(reward)""" next_action_batch = self.actor_network.target_action(next_state_batch) q_value_batch = self.critic_network.target_q(next_state_batch, next_action_batch) y_batch = [] for i in range(len(minibatch)): if done_batch[i]: y_batch.append(reward_batch[i]) else: y_batch.append(reward_batch[i] + GAMMA * q_value_batch[i]) y_batch = np.resize(y_batch, [BATCH_SIZE, 1]) """Update critic by minimizing the loss L (training)""" self.critic_network.train(y_batch, state_batch, action_batch) """Update the actor policy using the sampled gradient:""" action_batch_for_gradients = self.actor_network.actions(state_batch) q_gradient_batch = self.critic_network.gradients( state_batch, action_batch_for_gradients) self.actor_network.train(q_gradient_batch, state_batch) """Update the target networks""" self.actor_network.update_target() self.critic_network.update_target() #print("Training Finished") def noise_action(self, state): """Select action a_t according to the current policy and exploration noise""" action = self.actor_network.action(state) exp_noise = self.exploration_noise.noise() action += exp_noise # action[0] = np.clip(action[0], 0, 1) # action[1] = np.clip(action[1], -1, 1) return action def action(self, state): action = self.actor_network.action(state) # action[0] = np.clip(action[0], 0, 1) # action[1] = np.clip(action[1], -1, 1) return action def perceive(self, state, action, reward, next_state, done): """Store transition (s_t,a_t,r_t,s_{t+1}) in replay buffer""" self.replay_buffer.add(state, action, reward, next_state, done) """Store transitions to replay start size then start training""" # if self.replay_buffer.count() % 1000 == 0: # print("The buffer count is ", self.replay_buffer.count()) if self.replay_buffer.count() > REPLAY_START_SIZE: self.train() # self.atten_rate *= 0.99995 if not self.threading.is_alive(): self.threading = threading.Thread(target=self.train, name='LoopThread--DDPG') self.threading.start() """SAVE NETWORK""" if self.time_step % 100 == 0: print("Training_time_step:", self.time_step) if self.time_step % 1000 == 0: print("!!!!!!!save model success!!!!!!!!") self.actor_network.save_network(self.time_step) self.critic_network.save_network(self.time_step) """Re-iniitialize the random process when an episode ends""" if done: self.exploration_noise.reset()
class DDPG: def __init__(self, env, state_dim, action_dim): self.name = 'DDPG' self.environment = env self.time_step = 0 self.state_dim = state_dim self.action_dim = action_dim self.sess = tf.InteractiveSession() self.actor_network = ActorNetwork(self.sess, self.state_dim, self.action_dim) self.critic_network = CriticNetwork(self.sess, self.state_dim, self.action_dim) # initialize replay buffer self.replay_buffer = ReplayBuffer(REPLAY_BUFFER_SIZE) def train(self): minibatch = self.replay_buffer.get_batch(BATCH_SIZE) state_batch = np.asarray([data[0] for data in minibatch]) action_batch = np.asarray([data[1] for data in minibatch]) reward_batch = np.asarray([data[2] for data in minibatch]) next_state_batch = np.asarray([data[3] for data in minibatch]) done_batch = np.asarray([data[4] for data in minibatch]) # for action_dim = 1 action_batch = np.resize(action_batch, [BATCH_SIZE, self.action_dim]) next_action_batch = self.actor_network.target_actions(next_state_batch) q_value_batch = self.critic_network.target_q(next_state_batch, next_action_batch) y_batch = [] for i in range(len(minibatch)): if done_batch[i]: y_batch.append(reward_batch[i]) else: y_batch.append(reward_batch[i] + GAMMA * q_value_batch[i]) y_batch = np.resize(y_batch, [BATCH_SIZE, 1]) # Update critic by minimizing the loss L self.critic_network.train(y_batch, state_batch, action_batch) # Update the actor policy using the sampled gradient: action_batch_for_gradients = self.actor_network.actions(state_batch) q_gradient_batch = self.critic_network.gradients( state_batch, action_batch_for_gradients) self.actor_network.train(q_gradient_batch, state_batch) # Update the target networks self.actor_network.update_target() self.critic_network.update_target() def action(self, state): action = self.actor_network.action(state) return action def perceive(self, state, action, reward, next_state, done): # Store transition (s_t,a_t,r_t,s_{t+1}) in replay buffer self.replay_buffer.add(state, action, reward, next_state, done) if self.replay_buffer.count() == REPLAY_START_SIZE: print('\n---------------Start training---------------') # Store transitions to replay start size then start training if self.replay_buffer.count() > REPLAY_START_SIZE: self.time_step += 1 self.train() if self.time_step % 10000 == 0 and self.time_step > 0: self.actor_network.save_network(self.time_step) self.critic_network.save_network(self.time_step) return self.time_step
class DDPG: """docstring for DDPG""" def __init__(self, a_dim, s_dim): self.name = 'DDPG' # name for uploading results # self.environment = env # Randomly initialize actor network and critic network # with both their target networks self.state_dim = s_dim self.action_dim = a_dim self.time_step=0 self.max_bw = 0.0 self.max_cwnd = 0.0 self.min_rtt = 9999999.0 self.sess = tf.InteractiveSession() self.actor_network = ActorNetwork(self.sess, self.state_dim, self.action_dim) self.critic_network = CriticNetwork(self.sess, self.state_dim, self.action_dim) # initialize replay buffer self.replay_buffer = ReplayBuffer(REPLAY_BUFFER_SIZE) # Initialize a random process the Ornstein-Uhlenbeck process for action exploration self.exploration_noise = OUNoise(self.action_dim) def learn(self): # print "train step",self.time_step # Sample a random minibatch of N transitions from replay buffer minibatch = self.replay_buffer.get_batch(BATCH_SIZE) state_batch = np.asarray([data[0] for data in minibatch]) action_batch = np.asarray([data[1] for data in minibatch]) reward_batch = np.asarray([data[2] for data in minibatch]) next_state_batch = np.asarray([data[3] for data in minibatch]) done_batch = np.asarray([data[4] for data in minibatch]) # for action_dim = 1 action_batch = np.resize(action_batch, [BATCH_SIZE, self.action_dim]) # Calculate y_batch next_action_batch = self.actor_network.target_actions(next_state_batch) q_value_batch = self.critic_network.target_q(next_state_batch, next_action_batch) y_batch = [] for i in range(len(minibatch)): if done_batch[i]: y_batch.append(reward_batch[i]) else: y_batch.append(reward_batch[i] + GAMMA * q_value_batch[i]) y_batch = np.resize(y_batch, [BATCH_SIZE, 1]) # Update critic by minimizing the loss L self.critic_network.train(y_batch, state_batch, action_batch) # Update the actor policy using the sampled gradient: action_batch_for_gradients = self.actor_network.actions(state_batch) q_gradient_batch = self.critic_network.gradients(state_batch, action_batch_for_gradients) self.actor_network.train(q_gradient_batch, state_batch) # Update the target networks self.actor_network.update_target() self.critic_network.update_target() def noise_action(self, state): self.time_step += 1 # Select action a_t according to the current policy and exploration noise action = self.actor_network.action(state) noise = self.exploration_noise.noise() # print("noise:" + str(noise)) return action + noise def choose_action(self, state): self.time_step += 1 # print("_______________________choose_action_____________________") action = self.actor_network.action(state) return action def store_transition(self, s, a, r, s_,done,episode_count): # Store transition (s_t,a_t,r_t,s_{t+1}) in replay buffer # print("*********************************ADD****************************") self.replay_buffer.add(s, a, r, s_, done) # Store transitions to replay start size then start training if self.replay_buffer.count() > REPLAY_START_SIZE: if((episode_count+1)%100!= 0): self.learn() # print("learn!") else: self.actor_network.save_network(self.time_step) self.critic_network.save_network(self.time_step) # Re-iniitialize the random process when an episode ends if done: self.exploration_noise.reset() def extract_observation(self,dataRecorder,subflow_index,state_before): # print("extracting...") value_dic = dataRecorder.get_latest_data() state_after=state_before.reshape(10,5) # observation = np.zeros((4)) observation = np.zeros((5)) t_cWnd=[0,0] t_thr=[0,0] t_rtt=[0,0] t_loss_rate=[0,0] t_unAck=[0,0] s0=[0,0,0,0,0] state=np.zeros(1) for i in range(value_dic["nbOfSubflows"]): name = "cWnd" + str(i) t_cWnd[i] = value_dic[name] name = "rtt"+str(i) t_rtt[i] = value_dic[name] name = "unAck" + str(i) t_unAck[i]=value_dic[name] name = "loss_rate" + str(i) t_loss_rate[i]=value_dic[name] name = "throughput" + str(i) t_thr[i]=value_dic[name] thr=t_thr[subflow_index] s0[0]=t_thr[subflow_index] rtt=t_rtt[subflow_index] s0[1]=t_rtt[subflow_index] cwnd=t_cWnd[subflow_index] s0[2]=t_cWnd[subflow_index] loss_rate=t_loss_rate[subflow_index] s0[3]=t_loss_rate[subflow_index] unAck=t_unAck[subflow_index] s0[4]=t_unAck[subflow_index] s0=np.array(s0) min_=s0-s0 thr_n=s0[0] thr_n_min=s0[0]-min_[0] rtt_min=s0[1]-min_[1] cwnd_n_min=s0[2]-min_[2] loss_rate_n_min=s0[3]-min_[3] unAck_n_min=s0[4]-min_[4] # loss_rate_n_min=s0[7]-min_[7] if self.max_bw<thr_n_min: self.max_bw=thr_n_min if self.max_cwnd<cwnd_n_min: self.max_cwnd=cwnd_n_min if self.max_cwnd<cwnd_n_min: self.max_cwnd=cwnd_n_min if self.min_rtt>rtt_min: self.min_rtt=rtt_min reward = thr_n_min-5*(rtt_min-self.min_rtt)-10*loss_rate_n_min print("reward:"+str(reward)+" thr_n_min:"+str(thr_n_min)+ " rtt_min:"+str(rtt_min)+" self.min_rtt :"+str(self.min_rtt)+" delta_rtt"+str(rtt_min-self.min_rtt)) # print("unAck:"+str(unAck_n_min)) if self.max_bw!=0: state[0]=thr_n_min/self.max_bw # tmp=pacing_rate_n_min/self.max_bw state=np.append(state,[5*loss_rate_n_min]) state=np.append(state,[unAck_n_min]) else: state[0]=0 state=np.append(state,[0]) state=np.append(state,[0]) state=np.append(state,[1400/cwnd]) state=np.append(state,[self.min_rtt/rtt_min]) state_after=np.delete(state_after,[0],axis = 0) state_after=np.append(state_after,state) return state_after,reward,thr_n_min,rtt_min