def init_problem():
"""
Parses the Query statements and the Knowledge base
statements from the input file and returns them
"""
QUERIES = []
FOL_SENTENCES = []
with open(INPUT_FILE) as f_input:
file = list(f_input)
f_input.close()
NO_OF_QUERIES = int(file[0].rstrip('\n'))
for query_line in file[1:1 + NO_OF_QUERIES]:
query_line = query_line.rstrip()
query_line = query_line.replace(' ', '')
query_line = query_line.replace('\t', '')
QUERIES.append(Predicate(query_line))
NO_OF_FOL_SENTENCES = int(file[1 + NO_OF_QUERIES].rstrip('\n'))
for fol_sentence in file[2 + NO_OF_QUERIES:2 + NO_OF_QUERIES +
NO_OF_FOL_SENTENCES]:
fol_sentence = fol_sentence.rstrip()
fol_sentence = fol_sentence.replace(' ', '')
fol_sentence = fol_sentence.replace('\t', '')
FOL_SENTENCES.append(fol_sentence)
FOL_SENTENCES = list(set(FOL_SENTENCES))
return QUERIES, FOL_SENTENCES
def __init__(self, logic_stat_string=None):
if logic_stat_string:
pred_list = logic_stat_string.split('|')
pred_list = list(map(lambda x: Predicate(x), pred_list))
self.pred_set = set(pred_list)
self.logic_stat_string = '|'.join(
list(map(lambda x: x.pred_string, self.pred_set)))
else:
self.logic_stat_string, self.pred_set = None, None
def init_from_string(self, statement_string):
"""
initializes a Statement object from statement string
"""
predicate_list = statement_string.split('|')
predicate_list = map(lambda x: Predicate(x), predicate_list)
self.predicate_set = set(predicate_list)
statement_string_list = map(lambda x: x.predicate_string,
self.predicate_set)
self.statement_string = '|'.join(statement_string_list)
def __init__(self, statement_string=None):
if statement_string:
predicate_list = statement_string.split('|')
predicate_list = map(lambda x: Predicate(x), predicate_list)
self.predicate_set = set(predicate_list)
statement_string_list = map(lambda x: x.predicate_string,
self.predicate_set)
self.statement_string = '|'.join(statement_string_list)
else:
self.statement_string = None
self.predicate_set = None
def TestRun(controller, type, discrete_size, monsterMoveProb, isUpdate, trainingStage, objSet, maxStep, isEpisodeEnd, isShow, frameRate):
print "MaxStep: ", maxStep
size = 800, 800
gridSize = (discrete_size, discrete_size)
delay = 100
interval = 50
pygame.init()
pygame.key.set_repeat(delay, interval)
clock=pygame.time.Clock()
screen = pygame.display.set_mode(size)
actionList = ((0, 1), (0, -1), (1, 0), (-1, 0))
env = GridEnv.Grid((discrete_size, discrete_size), size, actionList, monsterMoveProb)
isTraining = not isEpisodeEnd
#maxStep = 200
numOfTurtle = objSet[0]
numOfCoin = objSet[1]
print "# coin ", numOfCoin
print "# Turtle ", numOfTurtle
print "training stage ", trainingStage
print "isEpisodeEnd ", isEpisodeEnd
count = 0
totalReward = 0
rewardList = []
stepCount = 0
while stepCount < maxStep:
#for i in range(0, maxEpisode):
#print totalReward
#rewardList[i] = totalReward
world = env.start(numOfTurtle, numOfCoin)
objLoc = getObjLoc(world, gridSize)
marioLoc = getMarioLoc(world, gridSize)
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.start(ob)
elif type == 'SARSA':
action = controller.start(Predicate.getSarsaFeature(ob))
#print Predicate.getSarsaFeature(ob)
elif type == 'LinearSARSA':
action = controller.start(Predicate.getSarsaFeature(ob))
count += 1
prevStepCount = stepCount
episodeReward = 0
while stepCount < maxStep:
stepCount = stepCount + 1
clock.tick(frameRate)
reward, world, flag = env.step(action, isTraining)
totalReward = totalReward + reward
episodeReward = episodeReward + reward
if flag:
if type == 'RRL':
controller.end(reward, trainingStage)
elif type == 'SARSA' or type == 'LinearSARSA':
controller.end(reward, isUpdate)
else:
assert False
break
objLoc = getObjLoc(world, gridSize)
marioLoc = getMarioLoc(world, gridSize)
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.step(reward, ob, trainingStage)
elif type == 'SARSA' or 'LinearSARSA':
action = controller.step(reward, Predicate.getSarsaFeature(ob), isUpdate)
else:
assert False
for event in pygame.event.get():
#action = 0
if event.type == pygame.QUIT: sys.exit()
if isShow:
screen.blit(env.getScreen(), (0, 0))
pygame.display.flip()
rewardList.append((prevStepCount, stepCount, episodeReward))
print totalReward
return rewardList, controller
def TestRun(controller, type, gridSize, monsterMoveProb, isUpdate, trainingStage, objSet, maxEpisode, isEpisodeEnd, isDraw, clockRate):
size = 800, 800
delay = 100
interval = 50
pygame.init()
pygame.key.set_repeat(delay, interval)
clock=pygame.time.Clock()
screen = pygame.display.set_mode(size)
actionList = controller.actionList
env = SimpleGridEnv.SimpleGrid(gridSize, size, actionList, monsterMoveProb)
isTraining = not isEpisodeEnd
maxStep = 50
numOfTurtle = objSet[0]
numOfCoin = objSet[1]
print "# coin ", numOfCoin
print "# Turtle ", numOfTurtle
print "training stage ", trainingStage
print "isEpisodeEnd ", isEpisodeEnd
count = 0
totalReward = 0
rewardList = {}
featureList = {}
#while 1:
for i in range(0, maxEpisode):
#print totalReward
rewardList[i] = totalReward
world = env.start(numOfTurtle, numOfCoin)
objLoc = getObjLoc(world, gridSize)
marioLoc = env.marioLoc
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.start(ob)
elif type == 'SARSA':
action = controller.start(Predicate.getSarsaFeature(ob))
count += 1
#if count % 100 == 0:
#print "monster------------------"
##print controller.agent[2].Q
#for y in range(-2, 3):
#for x in range(-2, 3):
#for action in actionList:
#controller.agent[2].touch((x, y), action)
#print (x, y), " ", action, " ", controller.agent[2].Q[((x, y), action)]
#print "coin------------------"
##print controller.agent[2].Q
#for y in range(-2, 3):
#for x in range(-2, 3):
#for action in actionList:
#controller.agent[3].touch((x, y), action)
#print (x, y), " ", action, " ", controller.agent[3].Q[((x, y), action)]
#print "world------------------"
#for y in range(0, 3):
#for x in range(0, 3):
#for action in actionList:
#controller.agent[1].touch((x, y), action)
#print (x, y), " ", action, " ", controller.agent[1].Q[((x, y), action)]
#print "all------------------"
#for y in range(0, 3):
#for x in range(0, 3):
#for action in actionList:
#controller.agent[0].touch((x, y), action)
#print (x, y), " ", action, " ", controller.agent[0].Q[((x, y), action)]
for j in range(0, maxStep):
clock.tick(clockRate)
reward, world, flag = env.step(action, isTraining)
#print objLoc
#print marioLoc
#print action
#print reward
matlabFeature = getMatlabFeature(reward, action, marioLoc, objLoc)
featureList.append(matlabFeature)
totalReward = totalReward + reward
if flag:
if type == 'RRL':
controller.end(reward, trainingStage)
elif type == 'SARSA':
controller.end(reward, isUpdate)
else:
assert False
break
objLoc = getObjLoc(world, gridSize)
marioLoc = env.marioLoc
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.step(reward, ob, trainingStage)
elif type == 'SARSA':
action = controller.step(reward, Predicate.getSarsaFeature(ob), isUpdate)
else:
assert False
for event in pygame.event.get():
#action = 0
if event.type == pygame.QUIT: sys.exit()
if isDraw:
screen.blit(env.getScreen(), (0, 0))
pygame.display.flip()
#print totalReward
return rewardList, controller, featureList
def TestRun(controller, type, discrete_size, monsterMoveProb, isUpdate,
trainingStage, objSet, maxStep, isEpisodeEnd, isShow, frameRate):
print "MaxStep: ", maxStep
size = 800, 800
gridSize = (discrete_size, discrete_size)
delay = 100
interval = 50
pygame.init()
pygame.key.set_repeat(delay, interval)
clock = pygame.time.Clock()
screen = pygame.display.set_mode(size)
actionList = ((0, 1), (0, -1), (1, 0), (-1, 0))
env = GridEnv.Grid((discrete_size, discrete_size), size, actionList,
monsterMoveProb)
isTraining = not isEpisodeEnd
#maxStep = 200
numOfTurtle = objSet[0]
numOfCoin = objSet[1]
print "# coin ", numOfCoin
print "# Turtle ", numOfTurtle
print "training stage ", trainingStage
print "isEpisodeEnd ", isEpisodeEnd
count = 0
totalReward = 0
rewardList = []
stepCount = 0
while stepCount < maxStep:
#for i in range(0, maxEpisode):
#print totalReward
#rewardList[i] = totalReward
world = env.start(numOfTurtle, numOfCoin)
objLoc = getObjLoc(world, gridSize)
marioLoc = getMarioLoc(world, gridSize)
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.start(ob)
elif type == 'SARSA':
action = controller.start(Predicate.getSarsaFeature(ob))
#print Predicate.getSarsaFeature(ob)
elif type == 'LinearSARSA':
action = controller.start(Predicate.getSarsaFeature(ob))
count += 1
prevStepCount = stepCount
episodeReward = 0
while stepCount < maxStep:
stepCount = stepCount + 1
clock.tick(frameRate)
reward, world, flag = env.step(action, isTraining)
totalReward = totalReward + reward
episodeReward = episodeReward + reward
if flag:
if type == 'RRL':
controller.end(reward, trainingStage)
elif type == 'SARSA' or type == 'LinearSARSA':
controller.end(reward, isUpdate)
else:
assert False
break
objLoc = getObjLoc(world, gridSize)
marioLoc = getMarioLoc(world, gridSize)
ob = (marioLoc, objLoc)
if type == 'RRL':
action = controller.step(reward, ob, trainingStage)
elif type == 'SARSA' or 'LinearSARSA':
action = controller.step(reward, Predicate.getSarsaFeature(ob),
isUpdate)
else:
assert False
for event in pygame.event.get():
#action = 0
if event.type == pygame.QUIT: sys.exit()
if isShow:
screen.blit(env.getScreen(), (0, 0))
pygame.display.flip()
rewardList.append((prevStepCount, stepCount, episodeReward))
print totalReward
return rewardList, controller
def SmallWorldTest(isShow, frameRate, discrete_size, worldConf, agentConf,
maxTrainEpisode, maxTestEpisode):
#isShow = True
#frameRate = 10
#discrete_size = 16
monsterMoveProb = 0.3
isUpdate = True
#worldConf = (3, 5)
actionList = ((0, 1), (0, -1), (1, 0), (-1, 0))
#if False:
dumpCount = 10000000000
initialQ = 0
#controller = LinearSARSA.LinearSARSA(0.1, 0.2, 0.9, actionList, initialQ, dumpCount)
#controller = SARSA.SARSA(0.1, 0.2, 0.9, actionList)
#reward, controller = TestRun(controller, 'LinearSARSA', discrete_size, monsterMoveProb, isUpdate, 4, worldConf, maxTrainEpisode, True, isShow, frameRate)
#SaveToCSV(reward, 'LinearSarsaTrainX'+ str(discrete_size) + 'X' + str(worldConf[0]) +'_' + str(worldConf[1])+ 'X' + str(maxTrainEpisode)+'.csv')
#reward, controller = TestRun(controller, 'LinearSARSA', discrete_size, monsterMoveProb, isUpdate, 4, worldConf, maxTestEpisode, True, isShow, frameRate)
#SaveToCSV(reward, 'LinearSarsaTestX'+ str(discrete_size) + 'X'+ str(worldConf[0]) +'_'+ str(worldConf[1])+'X' +str(maxTestEpisode)+'.csv')
#Save(controller, 'SarsaCompController'+'.txt')
#print controller.Q
#DumpSARSA(controller)
#preList = [Predicate.MarioPre(), Predicate.MonsterPre(), Predicate.CoinPre(), Predicate.CoinAndMonsterPre(), Predicate.RestPre()]
preList = []
for conf in agentConf:
preList.append(Predicate.CoinAndMonsterPre(conf[0], conf[1]))
preList.append(Predicate.RestPre())
controller = RelationalQ.RelationalQ(0.1, 0.2, 0.9, actionList, preList)
#trainEpisodeList = [0, 50, 100, 150]
trainEpisode = maxTrainEpisode / len(agentConf)
trainingStage = 1
lastConf = agentConf[len(agentConf) - 1]
print "Prepare training--------------"
for conf in agentConf: #don't run 2-order in the training stage
#print "stage: ", trainingStage
#print "monster: ", conf[0]
#print "coin: ", conf[1]
coinNum = conf[1]
isEpisodeEnd = False
if coinNum > 0:
isEpisodeEnd = True
reward, controller = TestRun(controller, 'RRL', discrete_size,
monsterMoveProb, isUpdate, trainingStage,
conf, trainEpisode, isEpisodeEnd, isShow,
frameRate)
#Save(controller, 'RRL_controller_train_'+str(trainEpisode)+ '_'+str(lastConf)+'_'+str(len(agentConf)) + '.txt')
#DumpRRL(controller)
trainingStage = trainingStage + 1
#DumpRRL(controller)
#raw_input("Press Enter to continue...")
#DumpRRL(controller)
isEpisodeEnd = True
reward, controller = TestRun(controller, 'RRL', discrete_size,
monsterMoveProb, isUpdate, trainingStage,
worldConf, maxTestEpisode, isEpisodeEnd,
isShow, frameRate)
SaveToCSV(
reward, 'RRLXtestX' + str(discrete_size) + 'X' + str(worldConf[0]) +
'_' + str(worldConf[1]) + 'X' + str(maxTrainEpisode) + 'X' +
str(lastConf[0]) + '_' + str(lastConf[1]) + 'X' + str(len(agentConf)) +
'.csv')
