def __init__(self, policy):
# world object, (starting state is trivial)
world = World((0,0),(1,1))
value = {}
for state in world.allStates():
value[state] = 0
discount = 0.9
delta = 1
while abs(delta) > 0.00001:
delta = 0
for state in world.allStates():
world.setState(state)
old = value[state]
# we can set the minimum to 0 since we know every value will be 0 or positive
curMax = 0
for move in world.moveList():
if world.posAfterMove(move) == (0,0):
probSum = 10
else:
probSum = 0
for nextState,prob in world.nextPreyStates():
probSum += prob*discount*value[nextState]
curMax = max(curMax,probSum)
value[state] = curMax
delta = max(delta,abs(old - curMax))
value[(0,0)] = 10
self.value = value
self.actionList = []
self.allList = []
self.bottomPolicy = policy
self.discount = discount
def Qlearning(episodes, policy, startState=(-5,-5), initValue=15,policyParam=0.1, alpha=0.4,discount=0.9):
# world object, (starting state is trivial)
world = World((0,0),(1,1))
# Q value table
Q = {}
for state in world.allStates():
for move in world.moveList():
Q[state,move] = initValue
steps = [0]*episodes
for i in range(episodes):
iterations = 0
# initialize world
world.setState(startState)
while True:
state = world.position
# move the predator according to policy with one parameter (epsilon for E-greedy or Tua for softmax)
action = policy(state, world, Q, policyParam)
world.move(action)
iterations += 1
# check if predator caught the prey
if world.stopState():
# the Q(s,a) update rule (note that the next state is the absorbing state)
Q[state,action] = Q[state,action] + alpha * (10 - Q[state,action])
break
# move the prey (stochasticly)
world.performPreyMove()
newState = world.position
# the maximum value the agent can have after another move
maxQ = max([Q[newState,nextAction] for nextAction in world.moveList()])
# the Q(s,a) update rule (note that the immediate reward is zero)
Q[state,action] = Q[state,action] + alpha * ( discount*maxQ - Q[state,action])
# print the number of steps the predator took
steps[i] = iterations
return steps
def isOptimal(self,state, move): world = World((0,0),(1,1)) ourMove = 0 bestMove = 0 for nmove in world.moveList(): world.setState(state) world.move(nmove) if world.position == (0,0): probSum = 10 else: probSum = 0 for nextState,prob in world.nextPreyStates(): probSum += prob*self.discount*self.value[nextState] bestMove = max(bestMove,probSum) if nmove == move: ourMove = probSum return ourMove/bestMove > 0.97
def MCon(episodes, initValue=15,epsilon=0.1, alpha=0.5,discount=0.9):
# world object, (starting state is trivial)
world = World((0,0),(1,1))
# initialize Q value table and Return list for every (s,a)-pair
Q = {}
R = {}
for state in world.allStates():
for move in world.moveList():
Q[state,move] = initValue # some value
R[state,move] = [] # empty list; return = cummulative discounted reward
steps = [0]*episodes # list counting number of iterations
for i in range(episodes):
iterations = 0
# initialize world
world.setState((-5,-5))
stateActionPairs = {}
# generate an episode using current policy
while True:
state = world.position
# move the predator according to policy
action = epsGreedyPolicy(state, world, Q, epsilon)
world.move(action)
if not (state,action) in stateActionPairs: # store first occurence
stateActionPairs[(state,action)] = iterations # will be used for discounting
iterations += 1
# check if predator caught the prey
if world.stopState():
break
# move the prey (stochasticly)
world.performPreyMove()
newState = world.position
steps[i] = iterations # save amount of iterations needed to catch the prey
# update Q and R
for pair in stateActionPairs.keys():
firstReturn = 10.0*discount**(iterations-stateActionPairs[pair]) # always zero but 10 when episode ends
R[pair].append(firstReturn)
Q[pair] = np.mean(np.array(R[pair]))
# update policy done in epsilon greedy policy code
return steps
def MCoff(episodes, behaPolicy, matches=[], initValue=15,discount=0.9):
# behaPolicy = dictionary with keys (state,action) and value P(action|state)
world = World((0,0),(1,1))
movelist = world.moveList()
def policy(world):
return world.pickElementWithProbs([(move,behaPolicy[(world.position,move)]) for move in movelist])
# initialize Q value table and Return list for every (s,a)-pair
Q = {}
R = {}
num = {}
denum = {}
for state in world.allStates():
for move in world.moveList():
num[state,move] = 0.0
denum[state,move] = 0.0
Q[state,move] = float(initValue) # some value
R[state,move] = [] # empty list; return = cummulative discounted reward
steps = [0]*episodes # list counting number of iterations
for epi in range(episodes):
time = 0
totalTime =0
# initialize world
world.setState((-5,-5))
episode = []
while True:
action = policy(world)
episode.append((world.position, action))
if action == None:
print action, state
world.move(action)
if world.stopState():
break
world.performPreyMove()
# save the pairs that match, and their first occurence
matchingHistory = {}
# last time move was equal to policy
last = 0
for i, (state, action) in enumerate(episode[::-1]):
actionValues = [(Q[state,maction],maction) for maction in world.moveList()]
bestActions = [actionValues[j][1] for j in maxIndices(actionValues)]
matchingHistory[(state, action)] = len(episode)-i - 1
if action not in bestActions:
last = len(episode)-i
break
matches.append(len(episode)-last)
for (state, action) in matchingHistory:
if matchingHistory[(state, action)] >= last-1:
w = np.prod([ 1.0/behaPolicy[episode[j]] for j in range(matchingHistory[(state, action)],len(episode))])
num[(state,move)] += w * (10.0*discount**matchingHistory[(state, action)]) # return is gamma^{T-t}*10
denum[(state,move)] += w
Q[(state,move)]= num[(state,move)]/float(denum[(state,move)])
world.setState((-5,-5))
iterations = 0
while True:
iterations += 1
actionValues = [(maction, Q[state,maction]) for maction in world.moveList()]
bestAction = random.choice([actionValues[j][0] for j in maxIndices(actionValues)])
world.move(bestAction)
if world.stopState() or iterations > 2000:
break
world.performPreyMove()
steps[epi] = iterations
return steps