class QNN(): """ Neural Q-Network. Includes experience replay. nactions: the number of actions input_size: the number of inputs max_experiences: the total number of experiences to save for replay gamma: future rewards discount rate alpha: learning rate for underlying NN use_sarsa: flag whether to use the SARSA update rule """ def __call__(self,s,a=None): """ implement here the returned Qvalue of state (s) and action(a) e.g. Q.GetValue(s,a) is equivalent to Q(s,a) """ if a==None: return self.GetValue(s) return self.GetValue(s,a) def __init__(self, nactions, input_size, max_experiences=500, gamma=0.6, alpha=0.1, use_sarsa=False): #Default uses 2 hidden layers of equal size lay = [input_size, int((nactions+input_size)/2.0), int((nactions+input_size)/2.0), nactions] self.nactions = nactions self.NN = NeuralNet(layers=lay, epsilon=0.154, learningRate=alpha) self.experiences = [] self.max_experiences = max_experiences self.gamma = gamma self.use_sarsa = use_sarsa self.prob_remember = 0.1 self.num_replay_samples = 10 def GetValue(self, s, a=None): """ Return the Q(s,a) value of state (s) for action (a) or all values for Q(s) """ out = self.NN.propagate(s) if (a==None): return out return out[a] def Update(self, s1, a1, r, s2, a2): """ update action value for action(a) """ if (self.use_sarsa): v = r + self.gamma*self.GetValue(s2, a2) else: v = r + self.gamma*max(self.GetValue(s2)) a = np.zeros(self.nactions) a[a1] = v self.NN.propagateAndUpdate(s1, a) def RememberExperience(self, s1, a1, r, s2, a2): if (random.random() < self.prob_remember): if (len(self.experiences) >= self.max_experiences): #TODO: Something more intelligent about how we determine what is worth forgetting self.experiences.pop(random.randint(0, self.max_experiences-1)) self.experiences.append(Experience(s1, a1, r, s2, a2)) def ExperienceReplay(self): #Skip until we have enough experience if (len(self.experiences) < self.num_replay_samples): return for i in xrange(self.num_replay_samples): index = random.randint(0, len(self.experiences)-1) exp = self.experiences[index] self.Update(exp.s1, exp.a1, exp.r, exp.s2, exp.a2)