def TestSim(obs):
MaxY = 16
MaxX = 22
state = WorldState(obs)
print "mario loc ", state.mario.x, " ", state.mario.y
commonVar = getCommonVar()
classVarList = getClassVar()
rewardVar = orange.FloatVariable("reward")
RewardLearner = Learner(commonVar, [rewardVar], 3000)
commonVar.pop(0)
DynamicLearner = Learner(commonVar, classVarList, 3000)
lastActionId = 9
modelFea = getModelFeature(state, [2.0, 1.0, 0.0, 0.0])
rewardFea = getTrainFeature(state, [0.0], lastActionId) # don't learn the pseudo reward
DynamicLearner.add([modelFea])
RewardLearner.add([rewardFea])
dynaLearner = [DynamicLearner for action in range(12)]
path = Optimize(state, dynaLearner, RewardLearner, 100, [], ActionRange)
newState = ExpandPath(path, MakeSimState(state, 10), dynaLearner, RewardLearner)
print type(newState)
print "hello"
for world in newState.worldList:
print "loc: ", world.mario.x
def agent_step(self, reward, obs):
#self.obsList.append(obs)
#if reward < -0.01 + epsilon and reward > -0.01 - epsilon:
#reward = -1
state = WorldState(obs)
fea = getSarsaFeature(state, self.lastAction)
lastMario = self.lastState.mario
mario = state.mario #for internal reward system
dx = mario.x - lastMario.x
reward = reward + dx
modelReward = 0
if isMarioInPit(state):
print "in pit !!!!!!!"
#reward = reward + InPitPenalty #no pit penalty for HORDQ
modelReward = InPitPenalty
if not self.isModelReady():
#fea = getSarsaFeature(obs)
action = self.agent.step(reward, fea, NoTask)
else:
#episilon greey policy
if random.random() < self.epsilon:
#select randomly
action = self.actionList[int(random.random()*len(self.actionList))]
print "random!!"
else:
possibleAction = self.agent.getPossibleAction(fea)
#if fea[0] == (): #if not monster around, pass control to the planner
#possibleAction = self.actionList
action = self.planning(state, possibleAction)
print "planning", action
self.agent.pseudoReward = 10000
action = self.agent.step(reward, fea, action)
self.agent.pseudoReward = self.initPseudoReward
#state.dump()
print "step loc:", self.stepNum, " ", mario.x , " ", mario.y, " ", mario.sx, " ", mario.sy
#state.path = []
#state.reward = 0
#nextState, isValid = ExpandPath([0], state, self.DynamicLearner, self.RewardLearner)
#nextState.dump()
#print "pred loc:", nextState.mario.x , " ", nextState.mario.y, " ", nextState.mario.sx, " ", nextState.mario.sy
#print "backoff reward: ", nextState.reward
#nextState, isValid = ExpandPath([action], state, self.DynamicLearner, self.RewardLearner)
#nextState.dump()
#print "pred loc:", nextState.mario.x , " ", nextState.mario.y, " ", nextState.mario.sx, " ", nextState.mario.sy
#print "pred rewar:", action, " ", nextState.reward
lastActionId = self.lastAction
deltaX = mario.x - (lastMario.x + lastMario.sx)
deltaY = mario.y - (lastMario.y + lastMario.sy)
aX = mario.sx - lastMario.sx
aY = mario.sy - lastMario.sy
classVar = [round(aX, Precision), round(aY, Precision), round(deltaX, Precision), round(deltaY, Precision)]
rewardClassVar = [round(modelReward, 0)]
modelFea = getModelFeature(self.lastState, classVar)
#rewardFea = getTrainFeature(self.lastState, rewardClassVar, lastActionId) #don't learn the pseudo reward
if self.isModelReady(): #TODO: too dirty
#predictModelClass = self.DynamicLearner[lastActionId].getClass(modelFea)
#predictModelClass = [round(v, 1) for v in predictModelClass]
#print "feature: ", lastActionId, " ", modelFea
#print "predict: ", predictModelClass
predictModelClass = self.DynamicLearner[lastActionId].getClass(modelFea)
predictModelClass = [round(v, 1) for v in predictModelClass]
roundClassVar = [round(v, 1) for v in classVar]
print "feature: ", lastActionId, " ", modelFea
print "predict: ", predictModelClass
if not roundClassVar == predictModelClass:
self.feaList[lastActionId].append(modelFea)
else:
print "pass model-------------"
else:
if not self.AgentType() == AgentType.SarsaAgent:
self.feaList[lastActionId].append(modelFea)
rewardFea = getRewardFeature(state, self.lastAction)
print "before pre reward: ", self.rewardAgent.getQ(rewardFea, action)
self.rewardAgent.step(rewardFea, modelReward, action)
print "pre reward: ", self.rewardAgent.getQ(rewardFea, action)
print "reward: ", modelReward
#if self.isModelReady():
#predictRewardClass = self.RewardLearner.getClass(rewardFea)
#predictRewardClass = [round(v, 0) for v in predictRewardClass]
#print "reward: ", modelReward
#print "pre reward: ", predictRewardClass
#if not rewardClassVar == predictRewardClass:
#self.rewardFeaList.append(rewardFea)
#else:
#print "pass reward-------------"
#else:
#if not self.AgentType() == SarsaAgent:
#self.rewardFeaList.append(rewardFea)
self.lastState = state
self.lastLastAction = self.lastAction
self.lastAction = action
self.stepNum = self.stepNum + 1
self.distList[len(self.distList)-1] = (self.totalStep + self.stepNum, self.lastState.mario.x, self.episodeNum, 0)
return makeAction(action)
