class PPO(Method): def __init__(self, sess, settings, netConfigOverride, stateShape, actionSize, nTrajs=1, **kwargs): """ Initializes a training method for a neural network. Parameters ---------- Model : Keras Model Object A Keras model object with fully defined layers and a call function. See examples in networks module. sess : Tensorflow Session Initialized Tensorflow session stateShape : list List of integers of the inputs shape size. Ex [39,39,6] actionSize : int Output size of the network. HPs : dict Dictionary that contains all hyperparameters to be used in the methods training nTrajs : int (Optional) Number that specifies the number of trajectories to be created for collecting training data. scope : str (Optional) Name of the PPO method. Used to group and differentiate variables between other networks. Returns ------- N/A """ #Processing inputs self.actionSize = actionSize self.sess = sess self.HPs = settings["NetworkHPs"] self.Model = NetworkBuilder(networkConfig=settings["NetworkConfig"], netConfigOverride=netConfigOverride, actionSize=actionSize) scope = "PPO" #Creating appropriate buffer for the method. self.buffer = [Trajectory(depth=8) for _ in range(nTrajs)] with self.sess.as_default(), self.sess.graph.as_default(): with tf.name_scope(scope): #Placeholders self.s = tf.placeholder(tf.float32, [None] + stateShape, 'S') self.a_his = tf.placeholder(tf.int32, [ None, ], 'A') self.td_target_ = tf.placeholder(tf.float32, [None], 'TD_target') self.advantage_ = tf.placeholder(shape=[None], dtype=tf.float32, name='adv_hold') self.old_log_logits_ = tf.placeholder(shape=[None, actionSize], dtype=tf.float32, name='old_logit_hold') #Initializing Netowrk I/O inputs = {"state": self.s} out = self.Model(inputs) self.a_prob = out["actor"] self.v = out["critic"] self.log_logits = out["log_logits"] # Entropy def _log(val): return tf.log(tf.clip_by_value(val, 1e-10, 10.0)) entropy = self.entropy = -tf.reduce_mean( self.a_prob * _log(self.a_prob), name='entropy') # Critic Loss td_error = self.td_target_ - self.v critic_loss = self.critic_loss = tf.reduce_mean( tf.square(td_error), name='critic_loss') # Actor Loss action_OH = tf.one_hot(self.a_his, actionSize, dtype=tf.float32) log_prob = tf.reduce_sum(self.log_logits * action_OH, 1) old_log_prob = tf.reduce_sum(self.old_log_logits_ * action_OH, 1) # Clipped surrogate function ratio = tf.exp(log_prob - old_log_prob) surrogate = ratio * self.advantage_ clipped_surrogate = tf.clip_by_value( ratio, 1 - self.HPs["eps"], 1 + self.HPs["eps"]) * self.advantage_ surrogate_loss = tf.minimum(surrogate, clipped_surrogate, name='surrogate_loss') actor_loss = self.actor_loss = -tf.reduce_mean( surrogate_loss, name='actor_loss') loss = self.actor_loss + self.critic_loss * self.HPs[ "CriticBeta"] # Build Trainer if self.HPs["Optimizer"] == "Adam": self.optimizerA = tf.keras.optimizers.Adam( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adam( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "RMS": self.optimizerA = tf.keras.optimizers.RMSProp( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.RMSProp( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adagrad": self.optimizerA = tf.keras.optimizers.Adagrad( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adagrad( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adadelta": self.optimizerA = tf.keras.optimizers.Adadelta( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adadelta( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adamax": self.optimizerA = tf.keras.optimizers.Adamax( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adamax( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Nadam": self.optimizerA = tf.keras.optimizers.Nadam( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Nadam( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "SGD": self.optimizerA = tf.keras.optimizers.SGD( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.SGD( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Amsgrad": self.optimizerA = tf.keras.optimizers.Nadam( self.HPs["LR Actor"], amsgrad=True) self.optimizerE = tf.keras.optimizers.Nadam( self.HPs["LR Entropy"], amsgrad=True) else: print("Not selected a proper Optimizer") exit() a_params = self.Model.GetVariables("Actor") c_params = self.Model.GetVariables("Critic") self.gradients_a = self.optimizerA.get_gradients( loss, self.Model.trainable_variables) # capped_gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in self.gradients_a] self.update_op_a = self.optimizerA.apply_gradients( zip(self.gradients_a, self.Model.trainable_variables)) entropy_loss = -self.entropy * self.HPs["EntropyBeta"] self.gradients_e = self.optimizerE.get_gradients( entropy_loss, a_params) self.update_op_e = self.optimizerE.apply_gradients( zip(self.gradients_e, a_params)) total_counter = 1 vanish_counter = 0 for gradient in self.gradients_a: total_counter += np.prod(gradient.shape) stuff = tf.reduce_sum( tf.cast( tf.math.less_equal(tf.math.abs(gradient), tf.constant(1e-8)), tf.int32)) vanish_counter += stuff self.vanishing_gradient = vanish_counter / total_counter self.update_ops = [self.update_op_a, self.update_op_e] self.logging_ops = [ self.actor_loss, self.critic_loss, self.entropy, tf.reduce_mean(self.advantage_), tf.reduce_mean(ratio), loss, self.vanishing_gradient ] self.labels = [ "Loss Actor", "Loss Critic", "Entropy", "Advantage", "PPO Ratio", "Loss Total", "Vanishing Gradient" ] self.logging_MA = [ MovingAverage(400) for i in range(len(self.logging_ops)) ] self.count_MA = MovingAverage(400) def GetAction(self, state, episode=1, step=0): """ Method to run data through the neural network. Parameters ---------- state : np.array Data with the shape of [N, self.stateShape] where N is number of smaples Returns ------- actions : list[int] List of actions based on NN output. extraData : list List of data that is passed to the execution code to be bundled with state data. """ try: probs, log_logits, v = self.sess.run( [self.a_prob, self.log_logits, self.v], {self.s: state}) except ValueError: probs, log_logits, v = self.sess.run( [self.a_prob, self.log_logits, self.v], {self.s: np.expand_dims(state, axis=0)}) actions = np.array([ np.random.choice(probs.shape[1], p=prob / sum(prob)) for prob in probs ]) confid = -np.mean(probs * np.log(probs), axis=1) if step == 0: self.store_actions = actions self.old_confid = confid self.count = 0 return actions, [v, log_logits, True] else: if confid < self.old_confid: # compare inverse entropy self.old_confid = confid self.store_actions = actions self.count_MA.append(self.count) self.count = 0 return actions, [v, log_logits, True] else: if self.count >= 4: self.old_confid = np.maximum( self.old_confid + self.HPs["ConfidenceAnnealing"], self.HPs["MinConfidence"]) self.count += 1 return self.store_actions, [v, log_logits, False] def Update(self, episode=0): """ Process the buffer and backpropagates the loses through the NN. Parameters ---------- HPs : dict Hyperparameters for training. Returns ------- N/A """ #Counting number of samples. samples = 0 for i in range(len(self.buffer)): samples += len(self.buffer[i]) if samples < self.HPs["BatchSize"]: return for traj in range(len(self.buffer)): td_target_hier, advantage_hier, actions_hier, ll_hier, s_hier = self.ProcessBuffer( traj) for epoch in range(self.HPs["Epochs"]): for batch in MultiBatchDivider([ s_hier, actions_hier, td_target_hier, advantage_hier, ll_hier ], self.HPs["MinibatchSize"]): #Staging Buffer inputs into the entries to run through the network. feedDict = { self.s: np.asarray(batch[0]).squeeze(), self.a_his: np.asarray(batch[1]).squeeze(), self.td_target_: np.asarray(batch[2]).squeeze(), self.advantage_: np.reshape(batch[3], [-1]), self.old_log_logits_: np.asarray(batch[4]).squeeze() } out = self.sess.run( self.update_ops + self.logging_ops, feedDict) # local grads applied to global net. logging = out[len(self.update_ops):] for i, log in enumerate(logging): self.logging_MA[i].append(log) self.ClearTrajectory() def GetStatistics(self): dict = {} for i, label in enumerate(self.labels): dict["Training Results/" + label] = self.logging_MA[i]() dict["Training Results/Average Traj Length"] = self.count_MA() return dict def ProcessBuffer(self, traj): """ Process the buffer and backpropagates the loses through the NN. Parameters ---------- Model : HPs Hyperparameters for training. traj : Trajectory Data stored by the neural network. Returns ------- td_target : list List Temporal Difference Target for particular states. advantage : list List of advantages for particular actions. """ # Split into different episodes based on the "done" signal. Assumes that episode terminates at done. # Cannot account for instances where there are multiple done signals in a row. split_loc = [i + 1 for i, x in enumerate(self.buffer[traj][4]) if x] # reward_lists = np.split(self.buffer[traj][2],split_loc) # value_lists = np.split(self.buffer[traj][5],split_loc) # # td_target=[]; advantage=[] # for rew,value in zip(reward_lists,value_lists): # td_target_i, advantage_i = gae(rew.reshape(-1).tolist(),value.reshape(-1).tolist(),0,self.HPs["Gamma"],self.HPs["lambda"]) # td_target.extend(td_target_i); advantage.extend( advantage_i) # return td_target, advantage reward_lists = np.split(self.buffer[traj][2], split_loc[:-1]) #Stuff needed for the HL_S_lists = np.split(self.buffer[traj][0], split_loc[:-1]) HL_Critic_lists = np.split(self.buffer[traj][5], split_loc[:-1]) HL_Logits_lists = np.split(self.buffer[traj][6], split_loc[:-1]) HL_action_lists = np.split(self.buffer[traj][1], split_loc[:-1]) HL_flag_lists = np.split(self.buffer[traj][7], split_loc[:-1]) td_target_hier = [] advantage_hier = [] ll = [] actions = [] s = [] for rew, HL_critic, HL_ll, HL_a, HL_flag, HL_s in zip( reward_lists, HL_Critic_lists, HL_Logits_lists, HL_action_lists, HL_flag_lists, HL_S_lists): #Colapsing different trajectory lengths for the hierarchical controller split_loc_ = [i for i, x in enumerate(HL_flag[:-1]) if x][1:] rew_hier = [np.sum(l) for l in np.split(rew, split_loc_)] value_hier = [l[0] for l in np.split(HL_critic, split_loc_)] actions.extend([l[0] for l in np.split(HL_a, split_loc_)]) ll.extend([l[0] for l in np.split(HL_ll, split_loc_)]) s.extend([l[0] for l in np.split(HL_s, split_loc_)]) #Calculating the td_target and advantage for the hierarchical controller. td_target_i_, advantage_i_ = gae( np.asarray(rew_hier).reshape(-1).tolist(), np.asarray(value_hier).reshape(-1).tolist(), 0, self.HPs["Gamma"], self.HPs["lambda"]) td_target_hier.extend(td_target_i_) advantage_hier.extend(advantage_i_) return td_target_hier, advantage_hier, actions, ll, s @property def getVars(self): return self.Model.getVars("PPO_Training")
class PPO(Method): def __init__(self, sess, settings, netConfigOverride, stateShape, actionSize, nTrajs=1, **kwargs): """ Initializes a training method for a neural network. Parameters ---------- Model : Keras Model Object A Keras model object with fully defined layers and a call function. See examples in networks module. sess : Tensorflow Session Initialized Tensorflow session stateShape : list List of integers of the inputs shape size. Ex [39,39,6] actionSize : int Output size of the network. HPs : dict Dictionary that contains all hyperparameters to be used in the methods training nTrajs : int (Optional) Number that specifies the number of trajectories to be created for collecting training data. scope : str (Optional) Name of the PPO method. Used to group and differentiate variables between other networks. Returns ------- N/A """ #Processing inputs self.actionSize = actionSize self.sess = sess self.HPs = settings["NetworkHPs"] #Building the network. self.Model = NetworkBuilder(networkConfig=settings["NetworkConfig"], netConfigOverride=netConfigOverride, actionSize=actionSize) #Creating appropriate buffer for the method. self.buffer = [Trajectory(depth=7) for _ in range(nTrajs)] with self.sess.as_default(), self.sess.graph.as_default(): with tf.name_scope("PPO"): #Placeholders if len(stateShape) == 4: self.s = tf.placeholder(tf.float32, [None] + stateShape[0:4], 'S') else: self.s = tf.placeholder(tf.float32, [None] + stateShape, 'S') self.a_his = tf.placeholder(tf.int32, [ None, ], 'A') self.td_target_ = tf.placeholder(tf.float32, [None], 'Vtarget') self.advantage_ = tf.placeholder(shape=[None], dtype=tf.float32, name='adv_hold') self.old_log_logits_ = tf.placeholder(shape=[None, actionSize], dtype=tf.float32, name='old_logit_hold') #Initializing Netowrk I/O inputs = {"state": self.s} out = self.Model(inputs) self.a_prob = out["actor"] self.v = out["critic"] self.log_logits = out["log_logits"] # Entropy def _log(val): return tf.log(tf.clip_by_value(val, 1e-10, 10.0)) self.entropy = -tf.reduce_mean(self.a_prob * _log(self.a_prob), name='entropy') # Critic Loss td_error = self.td_target_ - self.v self.critic_loss = tf.reduce_mean(tf.square(td_error), name='critic_loss') # Actor Loss action_OH = tf.one_hot(self.a_his, actionSize, dtype=tf.float32) log_prob = tf.reduce_sum(self.log_logits * action_OH, 1) old_log_prob = tf.reduce_sum(self.old_log_logits_ * action_OH, 1) # Clipped surrogate function ratio = tf.exp(log_prob - old_log_prob) surrogate = ratio * self.advantage_ clipped_surrogate = tf.clip_by_value( ratio, 1 - self.HPs["eps"], 1 + self.HPs["eps"]) * self.advantage_ surrogate_loss = tf.minimum(surrogate, clipped_surrogate, name='surrogate_loss') self.actor_loss = -tf.reduce_mean(surrogate_loss, name='actor_loss') loss = self.actor_loss + self.critic_loss * self.HPs[ "CriticBeta"] # Build Trainer if self.HPs["Optimizer"] == "Adam": self.optimizerA = tf.keras.optimizers.Adam( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adam( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "RMS": self.optimizerA = tf.keras.optimizers.RMSProp( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.RMSProp( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adagrad": self.optimizerA = tf.keras.optimizers.Adagrad( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adagrad( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adadelta": self.optimizerA = tf.keras.optimizers.Adadelta( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adadelta( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Adamax": self.optimizerA = tf.keras.optimizers.Adamax( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Adamax( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Nadam": self.optimizerA = tf.keras.optimizers.Nadam( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.Nadam( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "SGD": self.optimizerA = tf.keras.optimizers.SGD( self.HPs["LR Actor"]) self.optimizerE = tf.keras.optimizers.SGD( self.HPs["LR Entropy"]) elif self.HPs["Optimizer"] == "Amsgrad": self.optimizerA = tf.keras.optimizers.Nadam( self.HPs["LR Actor"], amsgrad=True) self.optimizerE = tf.keras.optimizers.Nadam( self.HPs["LR Entropy"], amsgrad=True) else: print("Not selected a proper Optimizer") exit() a_params = self.Model.GetVariables("Actor") c_params = self.Model.GetVariables("Critic") self.gradients_a = self.optimizerA.get_gradients( loss, self.Model.trainable_variables) self.update_op_a = self.optimizerA.apply_gradients( zip(self.gradients_a, self.Model.trainable_variables)) entropy_loss = -self.entropy * self.HPs["EntropyBeta"] self.gradients_e = self.optimizerE.get_gradients( entropy_loss, a_params) self.update_op_e = self.optimizerE.apply_gradients( zip(self.gradients_e, a_params)) total_counter = 1 vanish_counter = 0 for gradient in self.gradients_a: total_counter += np.prod(gradient.shape) stuff = tf.reduce_sum( tf.cast( tf.math.less_equal(tf.math.abs(gradient), tf.constant(1e-8)), tf.int32)) vanish_counter += stuff self.vanishing_gradient = vanish_counter / total_counter self.update_ops = [self.update_op_a, self.update_op_e] self.logging_ops = [ self.actor_loss, self.critic_loss, self.entropy, tf.reduce_mean(self.advantage_), tf.reduce_mean(ratio), loss, self.vanishing_gradient ] self.labels = [ "Loss Actor", "Loss Critic", "Entropy", "Advantage", "PPO Ratio", "Loss Total", "Vanishing Gradient" ] self.logging_MA = [ MovingAverage(400) for i in range(len(self.logging_ops)) ] def GetAction(self, state, episode=1, step=0): """ Method to run data through the neural network. Parameters ---------- state : np.array Data with the shape of [N, self.stateShape] where N is number of smaples Returns ------- actions : list[int] List of actions based on NN output. extraData : list List of data that is passed to the execution code to be bundled with state data. """ try: probs, log_logits, v = self.sess.run( [self.a_prob, self.log_logits, self.v], {self.s: state}) except ValueError: probs, log_logits, v = self.sess.run( [self.a_prob, self.log_logits, self.v], {self.s: np.expand_dims(state, axis=0)}) actions = np.array([ np.random.choice(probs.shape[1], p=prob / sum(prob)) for prob in probs ]) return actions, [v, log_logits] def Update(self, episode=0): """ Process the buffer and backpropagates the loses through the NN. Parameters ---------- HPs : dict Hyperparameters for training. Returns ------- N/A """ samples = 0 for i in range(len(self.buffer)): samples += len(self.buffer[i]) if samples < self.HPs["BatchSize"]: return for traj in range(len(self.buffer)): #Finding if there are more than 1 done in the sequence. Clipping values if required. td_target, advantage = self.ProcessBuffer(traj) batches = len( self.buffer[traj][0]) // self.HPs["MinibatchSize"] + 1 s = np.array_split(self.buffer[traj][0], batches) a_his = np.array_split( np.asarray(self.buffer[traj][1]).reshape(-1), batches) td_target_ = np.array_split(td_target, batches) advantage_ = np.array_split(np.reshape(advantage, [-1]), batches) old_log_logits_ = np.array_split( np.reshape(self.buffer[traj][6], [-1, self.actionSize]), batches) #Create a dictionary with all of the samples? #Use a sampler to feed the update operation? #Staging Buffer inputs into the entries to run through the network. # print(td_target) for epoch in range(self.HPs["Epochs"]): for i in range(batches): feedDict = { self.s: np.squeeze(np.asarray(s[i])), self.a_his: np.asarray(a_his[i]), self.td_target_: np.asarray(td_target_[i]), self.advantage_: np.asarray(advantage_[i]), self.old_log_logits_: np.asarray(old_log_logits_[i]) } out = self.sess.run( self.update_ops + self.logging_ops, feedDict) # local grads applied to global net. logging = out[len(self.update_ops):] for i, log in enumerate(logging): self.logging_MA[i].append(log) self.ClearTrajectory() def GetStatistics(self): dict = {} for i, label in enumerate(self.labels): dict["Training Results/" + label] = self.logging_MA[i]() return dict def ProcessBuffer(self, traj): """ Process the buffer and backpropagates the loses through the NN. Parameters ---------- Model : HPs Hyperparameters for training. traj : Trajectory Data stored by the neural network. clip : list[bool] List where the trajectory has finished. Returns ------- td_target : list List Temporal Difference Target for particular states. advantage : list List of advantages for particular actions. """ # print("Starting Processing Buffer\n") # tracker.print_diff() split_loc = [i + 1 for i, x in enumerate(self.buffer[traj][4]) if x] reward_lists = np.split(self.buffer[traj][2], split_loc) value_lists = np.split(self.buffer[traj][5], split_loc) td_target = [] advantage = [] for rew, value in zip(reward_lists, value_lists): td_target_i, advantage_i = gae(rew.reshape(-1), value.reshape(-1).tolist(), 0, self.HPs["Gamma"], self.HPs["lambda"]) td_target.extend(td_target_i) advantage.extend(advantage_i) return td_target, advantage @property def getVars(self): return self.Model.getVars("PPO_Training")
class AC(Method): def __init__(self,sess,settings,netConfigOverride,stateShape,actionSize,nTrajs=1,**kwargs): """ Initializes I/O placeholders used for Tensorflow session runs. Initializes and Actor and Critic Network to be used for the purpose of RL. """ #Creating appropriate buffer for the method. self.buffer = [Trajectory(depth=5) for _ in range(nTrajs)] #Placeholders self.sess=sess self.HPs = settings["NetworkHPs"] self.s = tf.placeholder(dtype=tf.float32, shape=[None]+stateShape, name="state") self.a = tf.placeholder(tf.int32, [None,1], "act") # self.td_error = tf.placeholder(tf.float32, None, "td_error") # TD_error self.v_ = tf.placeholder(tf.float32, [None, 1], "v_next") self.r = tf.placeholder(tf.float32, [None,1], 'r') #These need to be returned in the call function of a tf.keras.Model class. self.Model = NetworkBuilder(networkConfig=settings["NetworkConfig"],netConfigOverride=netConfigOverride,actionSize=actionSize) inputs = {"state":self.s} out = self.Model(inputs) self.acts_prob = out["actor"] self.critic = out["critic"] #Defining Training Operations which will be called in the Update Function. with tf.variable_scope('Update_Operation'): with tf.name_scope('squared_TD_error'): self.td_error = self.r + self.HPs["Gamma"] * self.v_ - self.critic self.c_loss = tf.reduce_mean(tf.square(self.td_error)) # TD_error = (r+gamma*V_next) - V_eval with tf.name_scope('train_critic'): self.c_params = self.Model.GetVariables("Critic") self.c_grads = tf.gradients(self.c_loss, self.c_params) self.update_c_op = tf.train.AdamOptimizer(self.HPs["Critic LR"]).apply_gradients(zip(self.c_grads, self.c_params)) with tf.name_scope('exp_v'): log_prob = tf.log(self.acts_prob + 1e-5) * tf.one_hot(self.a, actionSize, dtype=tf.float32) self.a_loss = -tf.reduce_mean(log_prob * self.td_error) # advantage (TD_error) guided loss with tf.name_scope('train_actor'): self.a_params = self.Model.GetVariables("Actor") print(self.a_params) self.a_grads = tf.gradients(self.a_loss, self.a_params) self.update_a_op = tf.train.AdamOptimizer(self.HPs["Actor LR"]).apply_gradients(zip(self.a_grads, self.a_params)) self.update_ops=[self.update_c_op,self.update_a_op] self.entropy = -tf.reduce_mean(self.acts_prob * _log(self.acts_prob), name='entropy') self.logging_ops = [self.a_loss,self.c_loss,self.entropy] self.labels = ["Loss Actor","Loss Critic","Entropy"] self.logging_MA = [MovingAverage(400) for i in range(len(self.logging_ops))] def GetAction(self, state, episode=0, step=0): """ Contains the code to run the network based on an input. """ try: s = state[np.newaxis, :] probs = self.sess.run(self.acts_prob, {self.s: s}) # get probabilities for all actions except ValueError: probs = self.sess.run(self.acts_prob, {self.s: state}) # get probabilities for all actions return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel()), [] # return a int def Update(self, episode=0): """ Takes an input buffer and applies the updates to the networks through gradient backpropagation """ samples=0 for i in range(len(self.buffer)): samples +=len(self.buffer[i]) if samples < 1: return for traj in range(len(self.buffer)): v_ = self.sess.run(self.critic, {self.s: np.vstack(self.buffer[traj][3])}) feedDict = {self.s: np.vstack(self.buffer[traj][0]), self.v_: v_, self.r: np.vstack(self.buffer[traj][2]), self.a:np.vstack(self.buffer[traj][1]) } out = self.sess.run(self.update_ops+self.logging_ops, feedDict) # local grads applied to global net. logging = out[len(self.update_ops):] for i,log in enumerate(logging): self.logging_MA[i].append(log) #Clear of reset the buffer. self.ClearTrajectory() def GetStatistics(self): dict ={} for i,label in enumerate(self.labels): dict["Training Results/" + label] = self.logging_MA[i]() return dict @property def getVars(self): return self.Model.getVars("PPO_Training")