class Trainer(object):
""" Trainer is responsible for setting up the players and game.
"""
def __init__(self, args, server_port=6001, coach_port=6002):
self._serverPort = server_port # The port the server is listening on
self._coachPort = coach_port # The coach port to talk with the server
self._logDir = args.logDir # Directory to store logs
self._record = args.record # Record states + actions
self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
# =============== COUNTERS =============== #
self._numFrames = 0 # Number of frames seen in HFO trials
self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
self._frame = 0 # Current frame id
self._lastTrialStart = -1 # Frame Id in which the last trial started
# =============== TRIAL RESULTS =============== #
self._numTrials = 0 # Total number of HFO trials
self._numGoals = 0 # Trials in which the offense scored a goal
self._numBallsCaptured = 0 # Trials in which defense captured the ball
self._numBallsOOB = 0 # Trials in which ball went out of bounds
self._numOutOfTime = 0 # Trials that ran out of time
# =============== AGENT =============== #
self._numAgents = args.offenseAgents + args.defenseAgents
self._offenseAgents = args.offenseAgents
self._defenseAgents = args.defenseAgents
self._agentPlayGoalie = args.agentPlayGoalie
self._agentReady = set([]) # Unums of ready agents
self._agentTeams = [] # Names of the teams the agents are playing for
self._agentNumInt = [] # List of agents internal team numbers
self._agentServerPort = args.port # Base Port for agent's server
# =============== MISC =============== #
self._offenseTeamName = '' # Name of the offensive team
self._defenseTeamName = '' # Name of the defensive team
self._isPlaying = False # Is a game being played?
self._done = False # Are we finished?
self._agentPopen = [] # Agent's processes
self._npcPopen = [] # NPC's processes
self._connectedPlayers = [] # List of connected players
self.initMsgHandlers()
def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
"""Launches a learning agent using the agent binary
Returns a Popen process object
"""
if play_offense:
assert self._numOffense > 0
team_name = self._offenseTeamName
self._agentTeams.append(team_name)
# First offense number is reserved for inactive offensive goalie
internal_player_num = agent_num + 1
self._agentNumInt.append(internal_player_num)
else:
assert self._numDefense > 0
team_name = self._defenseTeamName
self._agentTeams.append(team_name)
internal_player_num = agent_num
self._agentNumInt.append(internal_player_num)
binary_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'teams', 'base')
config_dir = os.path.join(binary_dir, 'config/formations-dt')
print(("Waiting for player-controlled agent %s-%d: config_dir=%s, "\
"server_port=%d, server_addr=%s, team_name=%s, play_goalie=%r"
% (self._offenseTeamName if play_offense else self._defenseTeamName,
agent_num, config_dir, self._serverPort, "localhost", team_name,
agent_ext_num==1)))
if wait_until_join:
self.waitOnPlayer(agent_ext_num, play_offense)
return None
def createTeam(self, requested_team_name, play_offense):
""" Given a team name, returns the team object. """
teams_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'teams')
if requested_team_name == 'helios':
print('Creating team Helios')
team_name = 'HELIOS_' + ('left' if play_offense else 'right')
team_dir = os.path.join(teams_dir, 'helios', 'helios-13Eindhoven')
lib_dir = os.path.join(teams_dir, 'helios', 'local', 'lib')
return Teams.Helios(team_name, team_dir, lib_dir,
binaryName='helios_player', host='localhost',
port=self._serverPort)
elif requested_team_name == 'base':
print('Creating team Agent2d (base)')
team_name = 'base_' + ('left' if play_offense else 'right')
team_dir = os.path.join(teams_dir, 'base')
lib_dir = None
return Teams.Agent2d(team_name, team_dir, lib_dir,
binaryName='sample_player', logDir=self._logDir,
record=self._record, host='localhost',
port=self._serverPort)
elif requested_team_name == 'robocin':
print('Creating team RoboCIN2d')
team_name = 'RoboCIn_' + ('left' if play_offense else 'right')
team_dir = os.path.join(teams_dir, 'robocin', 'src')
lib_dir = None
return Teams.RoboCIn(team_name, team_dir, lib_dir,
binaryName='robocin_player', logDir=self._logDir,
record=self._record, host='localhost',
port=self._serverPort)
else:
print('Unknown team requested: ' + requested_team_name)
sys.exit(1)
def getTeams(self, offense_team_name, defense_team_name):
""" Sets the offensive and defensive teams and player rosters. """
# Set up offense team
offenseTeam = self.createTeam(offense_team_name, play_offense=True)
self._offenseTeamName = offenseTeam._name
self._offenseOrder = [1] + offenseTeam._offense_order # 1 for goalie
# Set up defense team
defenseTeam = self.createTeam(defense_team_name, play_offense=False)
self._defenseTeamName = defenseTeam._name
self._defenseOrder = [1] + defenseTeam._defense_order # 1 for goalie
return (offenseTeam, defenseTeam)
def parseMsg(self, msg):
""" Parse a message """
assert(msg[0] == '(')
res, ind = self.__parseMsg(msg,1)
assert(ind == len(msg)), msg
return res
def __parseMsg(self, msg, ind):
""" Recursively parse a message. """
res = []
while True:
if msg[ind] == '"':
ind += 1
startInd = ind
while msg[ind] != '"':
ind += 1
res.append(msg[startInd:ind])
ind += 1
elif msg[ind] == '(':
inner,ind = self.__parseMsg(msg,ind+1)
res.append(inner)
elif msg[ind] == ' ':
ind += 1
elif msg[ind] == ')':
return res,ind+1
else:
startInd = ind
while msg[ind] not in ' ()':
ind += 1
res.append(msg[startInd:ind])
def initComm(self):
""" Initialize communication to server. """
self._comm = ClientCommunicator(port=self._coachPort)
self.send('(init (version 8.0))')
self.checkMsg('(init ok)', retryCount=5)
# self.send('(eye on)')
self.send('(ear on)')
def _hearRef(self, body):
""" Handles hear messages from referee. """
assert body[0] == 'referee', 'Expected referee message.'
_,ts,event = body
self._frame = int(ts)
endOfTrial = False
if 'GOAL' in event:
self._numGoals += 1
self._numGoalFrames += self._frame - self._lastTrialStart
endOfTrial = True
elif event == 'OUT_OF_BOUNDS':
self._numBallsOOB += 1
endOfTrial = True
elif 'CAPTURED_BY_DEFENSE' in event:
self._numBallsCaptured += 1
endOfTrial = True
elif event == 'OUT_OF_TIME':
self._numOutOfTime += 1
endOfTrial = True
elif event == 'HFO_FINISHED':
self._done = True
if endOfTrial:
self._numTrials += 1
print('EndOfTrial: %d / %d %d %s'%\
(self._numGoals, self._numTrials, self._frame, event))
self._numFrames += self._frame - self._lastTrialStart
self._lastTrialStart = self._frame
self.getConnectedPlayers()
def _hear(self, body):
""" Handle a hear message. """
if body[0] == 'referee':
self._hearRef(body)
return
timestep,playerInfo,msg = body
try:
_,team,player = playerInfo[:3]
length = int(msg[0])
except:
return
msg = msg[1:length+1]
if msg == 'START':
if self._isPlaying:
print('Already playing, ignoring message')
else:
self.startGame()
elif msg == 'DONE':
raise DoneError
elif msg == 'ready':
print('Agent Connected:', team, player)
self._agentReady.add((team, player))
else:
print('Unhandled message from agent: %s' % msg)
def initMsgHandlers(self):
""" Create handlers for different messages. """
self._msgHandlers = []
self.ignoreMsg('player_param')
self.ignoreMsg('player_type')
self.ignoreMsg('ok','change_mode')
self.ignoreMsg('ok','ear')
self.ignoreMsg('ok','move')
self.ignoreMsg('ok','recover')
self.ignoreMsg('ok','say')
self.ignoreMsg('server_param')
self.registerMsgHandler(self._hear,'hear')
def recv(self, retryCount=None):
""" Recieve a message. Retry a specified number of times. """
return self._comm.recvMsg(retryCount=retryCount).strip()
def send(self, msg):
""" Send a message. """
self._comm.sendMsg(msg)
def checkMsg(self, expectedMsg, retryCount=None):
""" Check that the next message is same as expected message. """
msg = self.recv(retryCount)
if msg != expectedMsg:
sys.stderr.write('Error with message')
sys.stderr.write(' expected: ' + expectedMsg)
sys.stderr.write(' received: ' + msg)
# print >>sys.stderr,len(expectedMsg),len(msg)
raise ValueError
def convertToExtPlayer(self, team, num):
""" Returns the external player number for a given player. """
assert team == self._offenseTeamName or team == self._defenseTeamName,\
'Invalid team name %s. Valid choices: %s, %s.'\
%(team, self._offenseTeamName, self._defenseTeamName)
if team == self._offenseTeamName:
return self._offenseOrder[num]
else:
return self._defenseOrder[num]
def registerMsgHandler(self,handler,*args,**kwargs):
'''Register a message handler.
Handler will be called on a message that matches *args.
'''
args = list(args)
i,_,_ = self._findHandlerInd(args)
if i < 0:
self._msgHandlers.append([args,handler])
else:
if ('quiet' not in kwargs) or (not kwargs['quiet']):
print('Updating handler for %s' % (' '.join(args)))
self._msgHandlers[i] = [args,handler]
def unregisterMsgHandler(self, *args):
""" Delete a message handler. """
i,_,_ = self._findHandlerInd(args)
assert(i >= 0)
del self._msgHandlers[i]
def _findHandlerInd(self, msg):
""" Find the handler for a particular message. """
msg = list(msg)
for i,(partial,handler) in enumerate(self._msgHandlers):
recPartial = msg[:len(partial)]
if recPartial == partial:
return i,len(partial),handler
return -1,None,None
def handleMsg(self, msg):
""" Handle a message using the registered handlers. """
i,prefixLength,handler = self._findHandlerInd(msg)
if i < 0:
print('Unhandled message:',msg[0:2])
else:
handler(msg[prefixLength:])
def ignoreMsg(self,*args,**kwargs):
""" Ignore a certain type of message. """
self.registerMsgHandler(lambda x: None,*args,**kwargs)
def listenAndProcess(self, retry_count=None):
""" Gather messages and process them. """
try:
msg = self.recv(retry_count)
assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
msg = self.parseMsg(msg)
self.handleMsg(msg)
except TimeoutError:
pass
def disconnectPlayer(self, player, player_num, on_offense):
"""Wait on a launched player to disconnect from the server. """
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
while (team_name, str(player_num)) in self._connectedPlayers:
self.send('(disconnect_player %s %d)'%(team_name, player_num))
self.getConnectedPlayers()
player.kill()
def getConnectedPlayers(self):
""" Get the list of connected players. Populates self._connectedPlayers. """
self._gotLook = False
self.send('(look)')
partial = ['ok','look']
def f(body):
self._gotLook = True
del self._connectedPlayers[:]
for i in range(4, len(body)):
_,team,num = body[i][0][:3]
if (team, num) not in self._connectedPlayers:
self._connectedPlayers.append((team,num))
self.registerMsgHandler(f,*partial,quiet=True)
while not self._gotLook:
self.listenAndProcess()
self.send('(look)')
self.ignoreMsg(*partial,quiet=True)
def waitOnPlayer(self, player_num, on_offense):
""" Wait on a launched player to connect and be reported by the server. """
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
while (team_name, str(player_num)) not in self._connectedPlayers:
self.getConnectedPlayers()
def allPlayersConnected(self):
""" Returns true all players are connected. """
return len(self._connectedPlayers) == self._numOffense + self._numDefense
def sendHFOConfig(self):
""" Broadcast the HFO configuration """
offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
for i in range(1, self._numOffense + 1)])
defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
for i in range(self._numDefense)])
self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
' num_defense %d offense_nums %s defense_nums %s)'
%(self._offenseTeamName, self._defenseTeamName,
self._numOffense, self._numDefense,
offense_nums, defense_nums))
def startGame(self):
""" Starts a game of HFO. """
self.send('(change_mode play_on)')
self._isPlaying = True
def printStats(self):
print('TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
%(self._numFrames,
self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan')))
print('Trials : %i' % self._numTrials)
print('Goals : %i' % self._numGoals)
print('Defense Captured : %i' % self._numBallsCaptured)
print('Balls Out of Bounds: %i' % self._numBallsOOB)
print('Out of Time : %i' % self._numOutOfTime)
def checkLive(self, necProcesses):
"""Returns true if each of the necessary processes is still alive and
running.
"""
for p,name in necProcesses:
if p is not None and p.poll() is not None:
print('Something necessary closed (%s), exiting' % name)
return False
return True
def run(self, necProcesses, offense_team_name, defense_team_name):
""" Run the trainer """
try:
(offenseTeam, defenseTeam) = self.getTeams(offense_team_name, defense_team_name)
offense_unums = self._offenseOrder[1: self._numOffense + 1]
sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
defense_unums = sorted(self._defenseOrder[: self._numDefense])
sorted_defense_agent_unums = \
defense_unums[:self._defenseAgents] if self._agentPlayGoalie \
else defense_unums[-self._defenseAgents:]
# Launch offense
agent_num = 0
for player_num in range(1, 12):
if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num
agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
play_offense=True, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
agent_num += 1
else:
player = offenseTeam.launch_npc(player_num)
self.waitOnPlayer(player_num, True)
if player_num in offense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'offense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=True)
# Launch defense
agent_num = 0
for player_num in range(1, 12):
if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num + self._offenseAgents
agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
play_offense=False, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
agent_num += 1
else:
player = defenseTeam.launch_npc(player_num)
self.waitOnPlayer(player_num, False)
if player_num in defense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'defense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=False)
print('Checking all players connected')
while not self.allPlayersConnected():
self.getConnectedPlayers()
time.sleep(0.1)
self.sendHFOConfig()
print('Starting game')
time.sleep(0.1)
self.startGame()
while self.allPlayersConnected() and self.checkLive(necProcesses) and not self._done:
prevFrame = self._frame
self.listenAndProcess()
except TimeoutError:
print('Haven\'t heard from the server for too long, Exiting')
except (KeyboardInterrupt, DoneError):
print('Finished')
finally:
try:
self._comm.sendMsg('(bye)')
except:
pass
for p in self._agentPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
for p in self._npcPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
self._comm.close()
self.printStats()
class Trainer(object):
""" Trainer is responsible for setting up the players and game.
"""
def __init__(self, args, rng=numpy.random.RandomState(), server_port=6001,
coach_port=6002):
self._rng = rng # The Random Number Generator
self._serverPort = server_port # The port the server is listening on
self._coachPort = coach_port # The coach port to talk with the server
self._logDir = args.logDir # Directory to store logs
self._record = args.record # Record states + actions
self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
self._maxTrials = args.numTrials # Maximum number of trials to play
self._maxFrames = args.numFrames # Maximum number of frames to play
self._maxFramesPerTrial = args.maxFramesPerTrial
# =============== FIELD DIMENSIONS =============== #
self.NUM_FRAMES_TO_HOLD = 2 # Hold ball this many frames to capture
self.HOLD_FACTOR = 1.5 # Gain to calculate ball control
self.PITCH_WIDTH = 68.0 # Width of the field
self.PITCH_LENGTH = 105.0 # Length of field in long-direction
self.UNTOUCHED_LENGTH = 100 # Trial will end if ball untouched for this long
# allowedBallX, allowedBallY defines the usable area of the playfield
self._allowedBallX = numpy.array([-0.1, 0.5 * self.PITCH_LENGTH])
self._allowedBallY = numpy.array([-0.5 * self.PITCH_WIDTH, 0.5 * self.PITCH_WIDTH])
# =============== COUNTERS =============== #
self._numFrames = 0 # Number of frames seen in HFO trials
self._frame = 0 # Current frame id
self._lastTrialStart = -1 # Frame Id in which the last trial started
self._lastFrameBallTouched = -1 # Frame Id in which ball was last touched
# =============== TRIAL RESULTS =============== #
self._numTrials = 0 # Total number of HFO trials
self._numGoals = 0 # Trials in which the offense scored a goal
self._numBallsCaptured = 0 # Trials in which defense captured the ball
self._numBallsOOB = 0 # Trials in which ball went out of bounds
self._numOutOfTime = 0 # Trials that ran out of time
# =============== AGENT =============== #
self._numAgents = args.offenseAgents + args.defenseAgents
self._offenseAgents = args.offenseAgents
self._defenseAgents = args.defenseAgents
self._agentTeams = [] # Names of the teams the agents are playing for
self._agentNumInt = [] # List of agents internal team numbers
self._agentNumExt = [] # List of agents external team numbers
self._agentServerPort = args.port # Base Port for agent's server
self._agentOnBall = args.agent_on_ball # If true, agent starts with the ball
# =============== MISC =============== #
self._offenseTeamName = '' # Name of the offensive team
self._defenseTeamName = '' # Name of the defensive team
self._playerPositions = numpy.zeros((11,2,2)) # Positions of the players
self._ballPosition = numpy.zeros(2) # Position of the ball
self._ballHeld = numpy.zeros((11,2)) # Track player holding the ball
self._teams = [] # Team indexes for offensive and defensive teams
self._SP = {} # Sever Parameters. Recieved when connecting to the server.
self._isPlaying = False # Is a game being played?
self._teamHoldingBall = None # Team currently in control of the ball
self._playerHoldingBall = None # Player current in control of ball
self._agentPopen = [] # Agent's processes
self.initMsgHandlers()
def launch_agent(self, agent_num, play_offense, port):
"""Launch the learning agent using the start_agent.sh script and
return a DummyPopen for the process.
"""
print '[Trainer] Launching Agent', str(agent_num)
if play_offense:
assert self._numOffense > 0
team_name = self._offenseTeamName
self._agentTeams.append(team_name)
# First offense number is reserved for inactive offensive goalie
internal_player_num = agent_num + 1
self._agentNumInt.append(internal_player_num)
numTeammates = self._numOffense - 1
numOpponents = self._numDefense
else:
assert self._numDefense > 0
team_name = self._defenseTeamName
self._agentTeams.append(team_name)
internal_player_num = agent_num
self._agentNumInt.append(internal_player_num)
numTeammates = self._numDefense - 1
numOpponents = self._numOffense
ext_num = self.convertToExtPlayer(team_name, internal_player_num)
self._agentNumExt.append(ext_num)
binary_dir = os.path.dirname(os.path.realpath(__file__))
agentCmd = 'start_agent.sh -t %s -u %i -p %i -P %i --log-dir %s'\
' --numTeammates %i --numOpponents %i'\
' --playingOffense %i --serverPort %i'\
%(team_name, ext_num, self._serverPort,
self._coachPort, self._logDir, numTeammates,
numOpponents, play_offense, port)
if self._record:
agentCmd += ' --record'
agentCmd = os.path.join(binary_dir, agentCmd)
agentCmd = agentCmd.split(' ')
# Ignore stderr because librcsc continually prints to it
kwargs = {}#{'stderr':open('/dev/null','w')}
p = subprocess.Popen(agentCmd, **kwargs)
p.wait()
pid_file = os.path.join(self._logDir, 'start%i'%p.pid)
print '[Trainer] Parsing agent\'s pid from file:', pid_file
assert os.path.isfile(pid_file)
with open(pid_file,'r') as f:
output = f.read()
pid = int(re.findall('PID: (\d+)',output)[0])
return DummyPopen(pid)
def getDefensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer defense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [9,10,8,11,7,4,6,2,3,5]
elif team_name == 'WrightEagle':
return [5,2,8,9,10,6,3,11,4,7]
else:
return [2,3,4,5,6,7,8,11,9,10]
def getOffensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer offense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [2,4,6,5,3,7,9,10,8,11]
elif team_name == 'WrightEagle':
return [11,4,7,3,6,10,8,9,2,5]
else:
return [11,7,8,9,10,6,3,2,4,5]
def setTeams(self):
""" Sets the offensive and defensive teams and player rosters. """
self._offenseTeamInd = 0
self._offenseTeamName = self._teams[self._offenseTeamInd]
self._defenseTeamName = self._teams[1-self._offenseTeamInd]
offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
self._offenseOrder = [1] + offensive_roster # 1 for goalie
self._defenseOrder = [1] + defensive_roster # 1 for goalie
def teamToInd(self, team_name):
""" Returns the index of a given team. """
return self._teams.index(team_name)
def parseMsg(self, msg):
""" Parse a message """
assert(msg[0] == '(')
res, ind = self.__parseMsg(msg,1)
assert(ind == len(msg)), msg
return res
def __parseMsg(self, msg, ind):
""" Recursively parse a message. """
res = []
while True:
if msg[ind] == '"':
ind += 1
startInd = ind
while msg[ind] != '"':
ind += 1
res.append(msg[startInd:ind])
ind += 1
elif msg[ind] == '(':
inner,ind = self.__parseMsg(msg,ind+1)
res.append(inner)
elif msg[ind] == ' ':
ind += 1
elif msg[ind] == ')':
return res,ind+1
else:
startInd = ind
while msg[ind] not in ' ()':
ind += 1
res.append(msg[startInd:ind])
def initComm(self):
""" Initialize communication to server. """
self._comm = ClientCommunicator(port=self._coachPort)
self.send('(init (version 8.0))')
self.checkMsg('(init ok)', retryCount=5)
# self.send('(eye on)')
self.send('(ear on)')
def _hear(self, body):
""" Handle a hear message. """
timestep,playerInfo,msg = body
if len(playerInfo) != 3:
return
_,team,player = playerInfo
try:
length = int(msg[0])
except:
return
msg = msg[1:length+1]
if msg == 'START':
if self._isPlaying:
print '[Trainer] Already playing, ignoring message'
else:
self.startGame()
elif msg == 'DONE':
raise DoneError
else:
print '[Trainer] Unhandled message from agent: %s' % msg
def initMsgHandlers(self):
""" Create handlers for different messages. """
self._msgHandlers = []
self.ignoreMsg('player_param')
self.ignoreMsg('player_type')
self.ignoreMsg('ok','change_mode')
self.ignoreMsg('ok','ear')
self.ignoreMsg('ok','move')
self.ignoreMsg('ok','recover')
self.ignoreMsg('ok','say')
self.registerMsgHandler(self._handleSP,'server_param')
self.registerMsgHandler(self._hear,'hear')
def recv(self, retryCount=None):
""" Recieve a message. Retry a specified number of times. """
return self._comm.recvMsg(retryCount=retryCount).strip()
def send(self, msg):
""" Send a message. """
self._comm.sendMsg(msg)
def checkMsg(self, expectedMsg, retryCount=None):
""" Check that the next message is same as expected message. """
msg = self.recv(retryCount)
if msg != expectedMsg:
print >>sys.stderr,'[Trainer] Error with message'
print >>sys.stderr,' expected: %s' % expectedMsg
print >>sys.stderr,' received: %s' % msg
# print >>sys.stderr,len(expectedMsg),len(msg)
raise ValueError
def extractPoint(self, msg):
""" Extract a point from the provided message. """
return numpy.array(map(float,msg[:2]))
def convertToExtPlayer(self, team, num):
""" Returns the external player number for a given player. """
assert team == self._offenseTeamName or team == self._defenseTeamName,\
'Invalid team name %s. Valid choices: %s, %s.'\
%(team, self._offenseTeamName, self._defenseTeamName)
if team == self._offenseTeamName:
return self._offenseOrder[num]
else:
return self._defenseOrder[num]
def convertFromExtPlayer(self, team, num):
""" Maps external player number to internal player number. """
if team == self._offenseTeamName:
return self._offenseOrder.index(num)
else:
return self._defenseOrder.index(num)
def seeGlobal(self, body):
"""Send a look message to extract global information on ball and
player positions.
"""
self.send('(look)')
self._frame = int(body[0])
for obj in body[1:]:
objType = obj[0]
objData = obj[1:]
if objType[0] == 'g':
continue
elif objType[0] == 'b':
self._ballPosition = self.extractPoint(objData)
elif objType[0] == 'p':
teamName = objType[1]
team = self.teamToInd(teamName)
playerNum = self.convertFromExtPlayer(teamName,int(objType[2]))
self._playerPositions[playerNum,:,team] = self.extractPoint(objData)
def registerMsgHandler(self,handler,*args,**kwargs):
'''Register a message handler.
Handler will be called on a message that matches *args.
'''
args = list(args)
i,_,_ = self._findHandlerInd(args)
if i < 0:
self._msgHandlers.append([args,handler])
else:
if ('quiet' not in kwargs) or (not kwargs['quiet']):
print '[Trainer] Updating handler for %s' % (' '.join(args))
self._msgHandlers[i] = [args,handler]
def unregisterMsgHandler(self, *args):
""" Delete a message handler. """
i,_,_ = self._findHandlerInd(args)
assert(i >= 0)
del self._msgHandlers[i]
def _findHandlerInd(self, msg):
""" Find the handler for a particular message. """
msg = list(msg)
for i,(partial,handler) in enumerate(self._msgHandlers):
recPartial = msg[:len(partial)]
if recPartial == partial:
return i,len(partial),handler
return -1,None,None
def handleMsg(self, msg):
""" Handle a message using the registered handlers. """
i,prefixLength,handler = self._findHandlerInd(msg)
if i < 0:
print '[Trainer] Unhandled message:',msg[0:2]
else:
handler(msg[prefixLength:])
def ignoreMsg(self,*args,**kwargs):
""" Ignore a certain type of message. """
self.registerMsgHandler(lambda x: None,*args,**kwargs)
def _handleSP(self, body):
""" Handler for the sever params message. """
for param in body:
try:
val = int(param[1])
except:
try:
val = float(param[1])
except:
val = param[1]
self._SP[param[0]] = val
def listenAndProcess(self):
""" Gather messages and process them. """
msg = self.recv()
assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
msg = self.parseMsg(msg)
self.handleMsg(msg)
def _readTeamNames(self,body):
""" Read the names of each of the teams. """
self._teams = []
for _,_,team in body:
self._teams.append(team)
time.sleep(0.1)
self.send('(team_names)')
def waitOnTeam(self, first):
"""Wait on a given team. First indicates if this is the first team
connected or the second.
"""
self.send('(team_names)')
partial = ['ok','team_names']
self.registerMsgHandler(self._readTeamNames,*partial,quiet=True)
while len(self._teams) < (1 if first else 2):
self.listenAndProcess()
#self.unregisterMsgHandler(*partial)
self.ignoreMsg(*partial,quiet=True)
def checkIfAllPlayersConnected(self):
""" Returns true if all players are connected. """
self.send('(look)')
partial = ['ok','look']
self._numPlayers = 0
def f(x):
self._numPlayers = len(x) - 4 # -4 for time, ball, goal_l, and goal_r
self.send('(look)')
self.registerMsgHandler(f,*partial)
while self._numPlayers != 2 * 11:
self.listenAndProcess()
self.ignoreMsg(*partial,quiet=True)
def startGame(self):
""" Starts a game of HFO. """
self.reset()
self.registerMsgHandler(self.seeGlobal, 'see_global')
self.registerMsgHandler(self.seeGlobal, 'ok', 'look', quiet=True)
#self.registerMsgHandler(self.checkBall,'ok','check_ball')
self.send('(look)')
self._isPlaying = True
def calcBallHolder(self):
'''Calculates the ball holder, returns results in teamInd, playerInd. '''
totalHeld = 0
for team in self._teams:
for i in range(11):
pos = self._playerPositions[i,:,self.teamToInd(team)]
distBound = self._SP['kickable_margin'] + self._SP['player_size'] \
+ self._SP['ball_size']
distBound *= self.HOLD_FACTOR
if numpy.linalg.norm(self._ballPosition - pos) < distBound:
self._ballHeld[i,self.teamToInd(team)] += 1
totalHeld += 1
else:
self._ballHeld[i,self.teamToInd(team)] = 0
# If multiple players are close to the ball, no-one is holding
if totalHeld > 1:
self._ballHeld[:,:] = 0
inds = numpy.transpose((self._ballHeld >= self.NUM_FRAMES_TO_HOLD).nonzero())
assert(len(inds) <= 1)
if len(inds) == 1:
return inds[0,1],inds[0,0]
else:
return None,None
def isGoal(self):
""" Returns true if a goal has been scored. """
return (self._ballPosition[0] > self._allowedBallX[1]) \
and (numpy.abs(self._ballPosition[1]) <= 0.5 * self._SP['goal_width'])
def isOOB(self):
""" Returns true if the ball is out of bounds. """
return self._ballPosition[0] < self._allowedBallX[0] \
or self._ballPosition[0] > self._allowedBallX[1] \
or self._ballPosition[1] < self._allowedBallY[0] \
or self._ballPosition[1] > self._allowedBallY[1]
def isCaptured(self):
""" Returns true if the ball is captured by defense. """
return self._teamHoldingBall not in [None,self._offenseTeamInd]
def isOOT(self):
""" Returns true if the trial has run out of time. """
return self._frame - self._lastFrameBallTouched > self.UNTOUCHED_LENGTH \
or (self._maxFramesPerTrial > 0 and self._frame -
self._lastTrialStart > self._maxFramesPerTrial)
def movePlayer(self, team, internal_num, pos, convertToExt=True):
""" Move a player to a specified position.
Args:
team: the team name of the player
interal_num: the player's internal number
pos: position to move player to
convertToExt: convert interal player num to external
"""
num = self.convertToExtPlayer(team, internal_num) if convertToExt \
else internal_num
self.send('(move (player %s %i) %f %f)' % (team, num, pos[0], pos[1]))
def moveBall(self, pos):
""" Moves the ball to a specified x,y position. """
self.send('(move (ball) %f %f 0.0 0.0 0.0)' % tuple(pos))
def randomPointInBounds(self, xBounds=None, yBounds=None):
"""Returns a random point inside of the box defined by xBounds,
yBounds. Where xBounds=[x_min, x_max] and yBounds=[y_min,
y_max]. Defaults to the xy-bounds of the playable HFO area.
"""
if xBounds is None:
xBounds = self.allowedBallX
if yBounds is None:
yBounds = self.allowedBallY
pos = numpy.zeros(2)
bounds = [xBounds, yBounds]
for i in range(2):
pos[i] = self._rng.rand() * (bounds[i][1] - bounds[i][0]) + bounds[i][0]
return pos
def boundPoint(self, pos):
"""Ensures a point is within the minimum and maximum bounds of the
HFO playing area.
"""
pos[0] = min(max(pos[0], self._allowedBallX[0]), self._allowedBallX[1])
pos[1] = min(max(pos[1], self._allowedBallY[0]), self._allowedBallY[1])
return pos
def reset(self):
""" Resets the HFO domain by moving the ball and players. """
self.resetBallPosition()
self.resetPlayerPositions()
self.send('(recover)')
self.send('(change_mode play_on)')
# self.send('(say RESET)')
def resetBallPosition(self):
"""Reset the position of the ball for a new HFO trial. """
self._ballPosition = self.boundPoint(self.randomPointInBounds(
.2*self._allowedBallX+.05*self.PITCH_LENGTH, .8*self._allowedBallY))
self.moveBall(self._ballPosition)
def getOffensiveResetPosition(self):
""" Returns a random position for an offensive player. """
offsets = [
[-1,-1],
[-1,1],
[1,1],
[1,-1],
[0,2],
[0,-2],
[-2,-2],
[-2,2],
[2,2],
[2,-2],
]
offset = offsets[self._rng.randint(len(offsets))]
offset_from_ball = 0.1 * self.PITCH_LENGTH * self._rng.rand(2) + \
0.1 * self.PITCH_LENGTH * numpy.array(offset)
return self.boundPoint(self._ballPosition + offset_from_ball)
# return self._ballPosition
def getDefensiveResetPosition(self):
""" Returns a random position for a defensive player. """
return self.boundPoint(self.randomPointInBounds(
[0.5 * 0.5 * self.PITCH_LENGTH, 0.75 * 0.5 * self.PITCH_LENGTH],
0.8 * self._allowedBallY))
def resetPlayerPositions(self):
"""Reset the positions of the players. This is called after a trial
ends to setup for the next trial.
"""
# Always Move the offensive goalie to the left goal
self.movePlayer(self._offenseTeamName, 0, [-0.5 * self.PITCH_LENGTH, 0])
# Move the rest of the offense
for i in xrange(1, self._numOffense + 1):
self.movePlayer(self._offenseTeamName, i, self.getOffensiveResetPosition())
# Move the agent to the ball
if self._agentOnBall and self._offenseAgents > 0:
self.movePlayer(self._offenseTeamName, self._agentNumInt[0], self._ballPosition)
# Move the defensive goalie
if self._numDefense > 0:
self.movePlayer(self._defenseTeamName, 0, [0.5 * self.PITCH_LENGTH,0])
# Move the rest of the defense
for i in xrange(1, self._numDefense):
self.movePlayer(self._defenseTeamName, i, self.getDefensiveResetPosition())
def removeNonHFOPlayers(self):
"""Removes players that aren't involved in HFO game.
The players whose numbers are greater than numOffense/numDefense
are sent to left-field.
"""
offensive_agent_numbers = self._agentNumInt[:self._offenseAgents]
defensive_agent_numbers = self._agentNumInt[self._offenseAgents:]
for i in xrange(self._numOffense + 1, 11):
if i not in offensive_agent_numbers:
self.movePlayer(self._offenseTeamName, i, [-0.25 * self.PITCH_LENGTH, 0])
for i in xrange(self._numDefense, 11):
if i not in defensive_agent_numbers:
self.movePlayer(self._defenseTeamName, i, [-0.25 * self.PITCH_LENGTH, 0])
def step(self):
""" Takes a simulated step. """
# self.send('(check_ball)')
self.removeNonHFOPlayers()
self._teamHoldingBall, self._playerHoldingBall = self.calcBallHolder()
if self._teamHoldingBall is not None:
self._lastFrameBallTouched = self._frame
if self.trialOver():
self.updateResults()
self._lastFrameBallTouched = self._frame
self.reset()
def updateResults(self):
""" Updates the various members after a trial has ended. """
if self.isGoal():
self._numGoals += 1
result = 'Goal'
self.send('(say GOAL)')
elif self.isOOB():
self._numBallsOOB += 1
result = 'Out of Bounds'
self.send('(say OUT_OF_BOUNDS)')
elif self.isCaptured():
self._numBallsCaptured += 1
result = 'Defense Captured'
self.send('(say CAPTURED_BY_DEFENSE)')
elif self.isOOT():
self._numOutOfTime += 1
result = 'Ball untouched for too long'
self.send('(say OUT_OF_TIME)')
else:
print '[Trainer] Error: Unable to detect reason for End of Trial!'
sys.exit(1)
print '[Trainer] EndOfTrial: %2i /%2i %5i %s'%\
(self._numGoals, self._numTrials, self._frame, result)
self._numTrials += 1
self._numFrames += self._frame - self._lastTrialStart
self._lastTrialStart = self._frame
if (self._maxTrials > 0) and (self._numTrials >= self._maxTrials):
raise DoneError
if (self._maxFrames > 0) and (self._numFrames >= self._maxFrames):
raise DoneError
def trialOver(self):
"""Returns true if the trial has ended for one of the following
reasons: Goal scored, out of bounds, captured by defense, or
untouched for too long.
"""
# The trial is still being setup, it cannot be over.
if self._frame - self._lastTrialStart < 5:
return False
return self.isGoal() or self.isOOB() or self.isCaptured() or self.isOOT()
def printStats(self):
print 'Num frames in completed trials : %i' % (self._numFrames)
print 'Trials : %i' % self._numTrials
print 'Goals : %i' % self._numGoals
print 'Defense Captured : %i' % self._numBallsCaptured
print 'Balls Out of Bounds: %i' % self._numBallsOOB
print 'Out of Time : %i' % self._numOutOfTime
def checkLive(self, necProcesses):
"""Returns true if each of the necessary processes is still alive and
running.
"""
for p,name in necProcesses:
if p.poll() is not None:
print '[Trainer] Something necessary closed (%s), exiting' % name
return False
return True
def run(self, necProcesses):
""" Run the trainer """
try:
for agent_num in xrange(self._offenseAgents):
port = self._agentServerPort + agent_num
agent = self.launch_agent(agent_num, play_offense=True, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
for agent_num in xrange(self._defenseAgents):
port = self._agentServerPort + agent_num + self._offenseAgents
agent = self.launch_agent(agent_num, play_offense=False, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
# Broadcast the HFO configuration
offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
for i in xrange(1, self._numOffense + 1)])
defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
for i in xrange(self._numDefense)])
self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
' num_defense %d offense_nums %s defense_nums %s)'
%(self._offenseTeamName, self._defenseTeamName,
self._numOffense, self._numDefense,
offense_nums, defense_nums))
self.startGame()
while self.checkLive(necProcesses):
prevFrame = self._frame
self.listenAndProcess()
if self._frame != prevFrame:
self.step()
except TimeoutError:
print '[Trainer] Haven\'t heard from the server for too long, Exiting'
except (KeyboardInterrupt, DoneError):
print '[Trainer] Exiting'
finally:
for p in self._agentPopen:
p.send_signal(SIGINT)
try:
self._comm.sendMsg('(bye)')
except:
pass
self._comm.close()
self.printStats()
class Trainer(object):
""" Trainer is responsible for setting up the players and game.
"""
def __init__(self, args, server_port=6001, coach_port=6002):
self._serverPort = server_port # The port the server is listening on
self._coachPort = coach_port # The coach port to talk with the server
self._logDir = args.logDir # Directory to store logs
self._record = args.record # Record states + actions
self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
# =============== COUNTERS =============== #
self._numFrames = 0 # Number of frames seen in HFO trials
self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
self._frame = 0 # Current frame id
self._lastTrialStart = -1 # Frame Id in which the last trial started
# =============== TRIAL RESULTS =============== #
self._numTrials = 0 # Total number of HFO trials
self._numGoals = 0 # Trials in which the offense scored a goal
self._numBallsCaptured = 0 # Trials in which defense captured the ball
self._numBallsOOB = 0 # Trials in which ball went out of bounds
self._numOutOfTime = 0 # Trials that ran out of time
# =============== AGENT =============== #
self._numAgents = args.offenseAgents + args.defenseAgents
self._offenseAgents = args.offenseAgents
self._defenseAgents = args.defenseAgents
self._agentReady = set([]) # Unums of ready agents
self._agentTeams = [] # Names of the teams the agents are playing for
self._agentNumInt = [] # List of agents internal team numbers
self._agentServerPort = args.port # Base Port for agent's server
# =============== MISC =============== #
self._offenseTeamName = '' # Name of the offensive team
self._defenseTeamName = '' # Name of the defensive team
self._isPlaying = False # Is a game being played?
self._done = False # Are we finished?
self._agentPopen = [] # Agent's processes
self._npcPopen = [] # NPC's processes
self._connectedPlayers = [] # List of connected players
self.initMsgHandlers()
def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
"""Launches a learning agent using the agent binary
Returns a Popen process object
"""
if play_offense:
assert self._numOffense > 0
team_name = self._offenseTeamName
self._agentTeams.append(team_name)
# First offense number is reserved for inactive offensive goalie
internal_player_num = agent_num + 1
self._agentNumInt.append(internal_player_num)
else:
assert self._numDefense > 0
team_name = self._defenseTeamName
self._agentTeams.append(team_name)
internal_player_num = agent_num
self._agentNumInt.append(internal_player_num)
binary_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'teams', 'base')
config_dir = os.path.join(binary_dir, 'config/formations-dt')
print("Waiting for player-controlled agent %s-%d: config_dir=%s, "\
"server_port=%d, server_addr=%s, team_name=%s, play_goalie=%r"
% (self._offenseTeamName if play_offense else self._defenseTeamName,
agent_num, config_dir, self._serverPort, "localhost", team_name,
agent_ext_num==1))
if wait_until_join:
self.waitOnPlayer(agent_ext_num, play_offense)
return None
def createTeam(self, requested_team_name, play_offense):
""" Given a team name, returns the team object. """
teams_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'teams')
if requested_team_name == 'helios':
print 'Creating team Helios'
team_name = 'HELIOS_' + ('left' if play_offense else 'right')
team_dir = os.path.join(teams_dir, 'helios', 'helios-13Eindhoven')
lib_dir = os.path.join(teams_dir, 'helios', 'local', 'lib')
return Teams.Helios(team_name, team_dir, lib_dir,
binaryName='helios_player', host='localhost',
port=self._serverPort)
elif requested_team_name == 'base':
print 'Creating team Agent2d (base)'
team_name = 'base_' + ('left' if play_offense else 'right')
team_dir = os.path.join(teams_dir, 'base')
lib_dir = None
return Teams.Agent2d(team_name, team_dir, lib_dir,
binaryName='sample_player', logDir=self._logDir,
record=self._record, host='localhost',
port=self._serverPort)
else:
print 'Unknown team requested: ' + requested_team_name
sys.exit(1)
def getTeams(self, offense_team_name, defense_team_name):
""" Sets the offensive and defensive teams and player rosters. """
# Set up offense team
offenseTeam = self.createTeam(offense_team_name, play_offense=True)
self._offenseTeamName = offenseTeam._name
self._offenseOrder = [1] + offenseTeam._offense_order # 1 for goalie
# Set up defense team
defenseTeam = self.createTeam(defense_team_name, play_offense=False)
self._defenseTeamName = defenseTeam._name
self._defenseOrder = [1] + defenseTeam._defense_order # 1 for goalie
return (offenseTeam, defenseTeam)
def parseMsg(self, msg):
""" Parse a message """
assert(msg[0] == '(')
res, ind = self.__parseMsg(msg,1)
assert(ind == len(msg)), msg
return res
def __parseMsg(self, msg, ind):
""" Recursively parse a message. """
res = []
while True:
if msg[ind] == '"':
ind += 1
startInd = ind
while msg[ind] != '"':
ind += 1
res.append(msg[startInd:ind])
ind += 1
elif msg[ind] == '(':
inner,ind = self.__parseMsg(msg,ind+1)
res.append(inner)
elif msg[ind] == ' ':
ind += 1
elif msg[ind] == ')':
return res,ind+1
else:
startInd = ind
while msg[ind] not in ' ()':
ind += 1
res.append(msg[startInd:ind])
def initComm(self):
""" Initialize communication to server. """
self._comm = ClientCommunicator(port=self._coachPort)
self.send('(init (version 8.0))')
self.checkMsg('(init ok)', retryCount=5)
# self.send('(eye on)')
self.send('(ear on)')
def _hearRef(self, body):
""" Handles hear messages from referee. """
assert body[0] == 'referee', 'Expected referee message.'
_,ts,event = body
self._frame = int(ts)
endOfTrial = False
if 'GOAL' in event:
self._numGoals += 1
self._numGoalFrames += self._frame - self._lastTrialStart
endOfTrial = True
elif event == 'OUT_OF_BOUNDS':
self._numBallsOOB += 1
endOfTrial = True
elif 'CAPTURED_BY_DEFENSE' in event:
self._numBallsCaptured += 1
endOfTrial = True
elif event == 'OUT_OF_TIME':
self._numOutOfTime += 1
endOfTrial = True
elif event == 'HFO_FINISHED':
self._done = True
if endOfTrial:
self._numTrials += 1
print 'EndOfTrial: %d / %d %d %s'%\
(self._numGoals, self._numTrials, self._frame, event)
self._numFrames += self._frame - self._lastTrialStart
self._lastTrialStart = self._frame
self.getConnectedPlayers()
def _hear(self, body):
""" Handle a hear message. """
if body[0] == 'referee':
self._hearRef(body)
return
timestep,playerInfo,msg = body
try:
_,team,player = playerInfo[:3]
length = int(msg[0])
except:
return
msg = msg[1:length+1]
if msg == 'START':
if self._isPlaying:
print 'Already playing, ignoring message'
else:
self.startGame()
elif msg == 'DONE':
raise DoneError
elif msg == 'ready':
print 'Agent Connected:', team, player
self._agentReady.add((team, player))
else:
print 'Unhandled message from agent: %s' % msg
def initMsgHandlers(self):
""" Create handlers for different messages. """
self._msgHandlers = []
self.ignoreMsg('player_param')
self.ignoreMsg('player_type')
self.ignoreMsg('ok','change_mode')
self.ignoreMsg('ok','ear')
self.ignoreMsg('ok','move')
self.ignoreMsg('ok','recover')
self.ignoreMsg('ok','say')
self.ignoreMsg('server_param')
self.registerMsgHandler(self._hear,'hear')
def recv(self, retryCount=None):
""" Recieve a message. Retry a specified number of times. """
return self._comm.recvMsg(retryCount=retryCount).strip()
def send(self, msg):
""" Send a message. """
self._comm.sendMsg(msg)
def checkMsg(self, expectedMsg, retryCount=None):
""" Check that the next message is same as expected message. """
msg = self.recv(retryCount)
if msg != expectedMsg:
print >>sys.stderr,'Error with message'
print >>sys.stderr,' expected: %s' % expectedMsg
print >>sys.stderr,' received: %s' % msg
# print >>sys.stderr,len(expectedMsg),len(msg)
raise ValueError
def convertToExtPlayer(self, team, num):
""" Returns the external player number for a given player. """
assert team == self._offenseTeamName or team == self._defenseTeamName,\
'Invalid team name %s. Valid choices: %s, %s.'\
%(team, self._offenseTeamName, self._defenseTeamName)
if team == self._offenseTeamName:
return self._offenseOrder[num]
else:
return self._defenseOrder[num]
def registerMsgHandler(self,handler,*args,**kwargs):
'''Register a message handler.
Handler will be called on a message that matches *args.
'''
args = list(args)
i,_,_ = self._findHandlerInd(args)
if i < 0:
self._msgHandlers.append([args,handler])
else:
if ('quiet' not in kwargs) or (not kwargs['quiet']):
print 'Updating handler for %s' % (' '.join(args))
self._msgHandlers[i] = [args,handler]
def unregisterMsgHandler(self, *args):
""" Delete a message handler. """
i,_,_ = self._findHandlerInd(args)
assert(i >= 0)
del self._msgHandlers[i]
def _findHandlerInd(self, msg):
""" Find the handler for a particular message. """
msg = list(msg)
for i,(partial,handler) in enumerate(self._msgHandlers):
recPartial = msg[:len(partial)]
if recPartial == partial:
return i,len(partial),handler
return -1,None,None
def handleMsg(self, msg):
""" Handle a message using the registered handlers. """
i,prefixLength,handler = self._findHandlerInd(msg)
if i < 0:
print 'Unhandled message:',msg[0:2]
else:
handler(msg[prefixLength:])
def ignoreMsg(self,*args,**kwargs):
""" Ignore a certain type of message. """
self.registerMsgHandler(lambda x: None,*args,**kwargs)
def listenAndProcess(self, retry_count=None):
""" Gather messages and process them. """
try:
msg = self.recv(retry_count)
assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
msg = self.parseMsg(msg)
self.handleMsg(msg)
except TimeoutError:
pass
def disconnectPlayer(self, player, player_num, on_offense):
"""Wait on a launched player to disconnect from the server. """
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
while (team_name, str(player_num)) in self._connectedPlayers:
self.send('(disconnect_player %s %d)'%(team_name, player_num))
self.getConnectedPlayers()
player.kill()
def getConnectedPlayers(self):
""" Get the list of connected players. Populates self._connectedPlayers. """
self._gotLook = False
self.send('(look)')
partial = ['ok','look']
def f(body):
self._gotLook = True
del self._connectedPlayers[:]
for i in xrange(4, len(body)):
_,team,num = body[i][0][:3]
if (team, num) not in self._connectedPlayers:
self._connectedPlayers.append((team,num))
self.registerMsgHandler(f,*partial,quiet=True)
while not self._gotLook:
self.listenAndProcess()
self.send('(look)')
self.ignoreMsg(*partial,quiet=True)
def waitOnPlayer(self, player_num, on_offense):
""" Wait on a launched player to connect and be reported by the server. """
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
while (team_name, str(player_num)) not in self._connectedPlayers:
self.getConnectedPlayers()
def allPlayersConnected(self):
""" Returns true all players are connected. """
return len(self._connectedPlayers) == self._numOffense + self._numDefense
def sendHFOConfig(self):
""" Broadcast the HFO configuration """
offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
for i in xrange(1, self._numOffense + 1)])
defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
for i in xrange(self._numDefense)])
self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
' num_defense %d offense_nums %s defense_nums %s)'
%(self._offenseTeamName, self._defenseTeamName,
self._numOffense, self._numDefense,
offense_nums, defense_nums))
def startGame(self):
""" Starts a game of HFO. """
self.send('(change_mode play_on)')
self._isPlaying = True
def printStats(self):
print 'TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
%(self._numFrames,
self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan'))
print 'Trials : %i' % self._numTrials
print 'Goals : %i' % self._numGoals
print 'Defense Captured : %i' % self._numBallsCaptured
print 'Balls Out of Bounds: %i' % self._numBallsOOB
print 'Out of Time : %i' % self._numOutOfTime
def checkLive(self, necProcesses):
"""Returns true if each of the necessary processes is still alive and
running.
"""
for p,name in necProcesses:
if p is not None and p.poll() is not None:
print 'Something necessary closed (%s), exiting' % name
return False
return True
def run(self, necProcesses, offense_team_name, defense_team_name):
""" Run the trainer """
try:
(offenseTeam, defenseTeam) = self.getTeams(offense_team_name, defense_team_name)
offense_unums = self._offenseOrder[1: self._numOffense + 1]
sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
defense_unums = self._defenseOrder[: self._numDefense]
sorted_defense_agent_unums = sorted(self._defenseOrder[:self._defenseAgents])
# Launch offense
agent_num = 0
for player_num in xrange(1, 12):
if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num
agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
play_offense=True, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
agent_num += 1
else:
player = offenseTeam.launch_npc(player_num)
self.waitOnPlayer(player_num, True)
if player_num in offense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'offense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=True)
# Launch defense
agent_num = 0
for player_num in xrange(1, 12):
if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num + self._offenseAgents
agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
play_offense=False, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
agent_num += 1
else:
player = defenseTeam.launch_npc(player_num)
self.waitOnPlayer(player_num, False)
if player_num in defense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'defense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=False)
print 'Checking all players connected'
while not self.allPlayersConnected():
self.getConnectedPlayers()
time.sleep(0.1)
self.sendHFOConfig()
print 'Starting game'
time.sleep(0.1)
self.startGame()
while self.allPlayersConnected() and self.checkLive(necProcesses) and not self._done:
prevFrame = self._frame
self.listenAndProcess()
except TimeoutError:
print 'Haven\'t heard from the server for too long, Exiting'
except (KeyboardInterrupt, DoneError):
print 'Finished'
finally:
try:
self._comm.sendMsg('(bye)')
except:
pass
for p in self._agentPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
for p in self._npcPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
self._comm.close()
self.printStats()
class Trainer(object):
""" Trainer is responsible for setting up the players and game.
"""
def __init__(self, args, server_port=6001, coach_port=6002):
self._serverPort = server_port # The port the server is listening on
self._coachPort = coach_port # The coach port to talk with the server
self._logDir = args.logDir # Directory to store logs
self._record = args.record # Record states + actions
self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
# =============== COUNTERS =============== #
self._numFrames = 0 # Number of frames seen in HFO trials
self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
self._frame = 0 # Current frame id
self._lastTrialStart = -1 # Frame Id in which the last trial started
# =============== TRIAL RESULTS =============== #
self._numTrials = 0 # Total number of HFO trials
self._numGoals = 0 # Trials in which the offense scored a goal
self._numBallsCaptured = 0 # Trials in which defense captured the ball
self._numBallsOOB = 0 # Trials in which ball went out of bounds
self._numOutOfTime = 0 # Trials that ran out of time
# =============== AGENT =============== #
self._numAgents = args.offenseAgents + args.defenseAgents
self._offenseAgents = args.offenseAgents
self._defenseAgents = args.defenseAgents
self._agentReady = set([]) # Unums of ready agents
self._agentTeams = [] # Names of the teams the agents are playing for
self._agentNumInt = [] # List of agents internal team numbers
self._agentServerPort = args.port # Base Port for agent's server
# =============== MISC =============== #
self._offenseTeamName = '' # Name of the offensive team
self._defenseTeamName = '' # Name of the defensive team
self._teams = [] # Team indexes for offensive and defensive teams
self._isPlaying = False # Is a game being played?
self._done = False # Are we finished?
self._agentPopen = [] # Agent's processes
self._npcPopen = [] # NPC's processes
self._connectedPlayers = []
self.initMsgHandlers()
def launch_npc(self, player_num, play_offense, wait_until_join=True):
"""Launches a player using sample_player binary
Returns a Popen process object
"""
print 'Launch npc %s-%d'%(self._offenseTeamName if play_offense
else self._defenseTeamName, player_num)
if play_offense:
team_name = self._offenseTeamName
else:
team_name = self._defenseTeamName
binary_dir = os.path.dirname(os.path.realpath(__file__))
config_dir = os.path.join(binary_dir, '../config/formations-dt')
player_conf = os.path.join(binary_dir, '../config/player.conf')
player_cmd = os.path.join(binary_dir, 'sample_player')
player_cmd += ' -t %s -p %i --config_dir %s ' \
' --log_dir %s --player-config %s' \
%(team_name, self._serverPort, config_dir,
self._logDir, player_conf)
if self._record:
player_cmd += ' --record'
if player_num == 1:
player_cmd += ' -g'
kwargs = {'stdout':open('/dev/null', 'w'),
'stderr':open('/dev/null', 'w')}
p = subprocess.Popen(player_cmd.split(' '), shell = False, **kwargs)
if wait_until_join:
self.waitOnPlayer(player_num, play_offense)
return p
def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
"""Launches a learning agent using the agent binary
Returns a Popen process object
"""
print 'Launch agent %s-%d'%(self._offenseTeamName if play_offense
else self._defenseTeamName, agent_num)
if play_offense:
assert self._numOffense > 0
team_name = self._offenseTeamName
self._agentTeams.append(team_name)
# First offense number is reserved for inactive offensive goalie
internal_player_num = agent_num + 1
self._agentNumInt.append(internal_player_num)
numTeammates = self._numOffense - 1
numOpponents = self._numDefense
else:
assert self._numDefense > 0
team_name = self._defenseTeamName
self._agentTeams.append(team_name)
internal_player_num = agent_num
self._agentNumInt.append(internal_player_num)
numTeammates = self._numDefense - 1
numOpponents = self._numOffense
binary_dir = os.path.dirname(os.path.realpath(__file__))
config_dir = os.path.join(binary_dir, '../config/formations-dt')
player_conf = os.path.join(binary_dir, '../config/player.conf')
agent_cmd = os.path.join(binary_dir, 'agent')
agent_cmd += ' -t %s -p %i --numTeammates %i --numOpponents %i' \
' --playingOffense %i --serverPort %i --log_dir %s' \
' --player-config %s --config_dir %s' \
%(team_name, self._serverPort, numTeammates,
numOpponents, play_offense, port, self._logDir,
player_conf, config_dir)
if agent_ext_num == 1:
agent_cmd += ' -g'
if self._record:
agent_cmd += ' --record'
# Comment next two lines to show output from agent.cpp and the server
kwargs = {'stdout':open('/dev/null', 'w'),
'stderr':open('/dev/null', 'w')}
p = subprocess.Popen(agent_cmd.split(' '), shell = False, **kwargs)
if wait_until_join:
self.waitOnPlayer(agent_ext_num, play_offense)
return p
def getDefensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer defense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [9,10,8,11,7,4,6,2,3,5]
elif team_name == 'WrightEagle':
return [5,2,8,9,10,6,3,11,4,7]
else:
return [2,3,4,5,6,7,8,11,9,10]
def getOffensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer offense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [2,4,6,5,3,7,9,10,8,11]
elif team_name == 'WrightEagle':
return [11,4,7,3,6,10,8,9,2,5]
else:
return [11,7,8,9,10,6,3,2,4,5]
def addTeam(self, team_name):
""" Adds a team to the team list"""
self._teams.append(team_name)
def setTeams(self):
""" Sets the offensive and defensive teams and player rosters. """
self._offenseTeamInd = 0
self._offenseTeamName = self._teams[self._offenseTeamInd]
self._defenseTeamName = self._teams[1-self._offenseTeamInd]
offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
self._offenseOrder = [1] + offensive_roster # 1 for goalie
self._defenseOrder = [1] + defensive_roster # 1 for goalie
def teamToInd(self, team_name):
""" Returns the index of a given team. """
return self._teams.index(team_name)
def parseMsg(self, msg):
""" Parse a message """
assert(msg[0] == '(')
res, ind = self.__parseMsg(msg,1)
assert(ind == len(msg)), msg
return res
def __parseMsg(self, msg, ind):
""" Recursively parse a message. """
res = []
while True:
if msg[ind] == '"':
ind += 1
startInd = ind
while msg[ind] != '"':
ind += 1
res.append(msg[startInd:ind])
ind += 1
elif msg[ind] == '(':
inner,ind = self.__parseMsg(msg,ind+1)
res.append(inner)
elif msg[ind] == ' ':
ind += 1
elif msg[ind] == ')':
return res,ind+1
else:
startInd = ind
while msg[ind] not in ' ()':
ind += 1
res.append(msg[startInd:ind])
def initComm(self):
""" Initialize communication to server. """
self._comm = ClientCommunicator(port=self._coachPort)
self.send('(init (version 8.0))')
self.checkMsg('(init ok)', retryCount=5)
# self.send('(eye on)')
self.send('(ear on)')
def _hearRef(self, body):
""" Handles hear messages from referee. """
assert body[0] == 'referee', 'Expected referee message.'
_,ts,event = body
self._frame = int(ts)
endOfTrial = False
if 'GOAL' in event:
self._numGoals += 1
self._numGoalFrames += self._frame - self._lastTrialStart
endOfTrial = True
elif event == 'OUT_OF_BOUNDS':
self._numBallsOOB += 1
endOfTrial = True
elif 'CAPTURED_BY_DEFENSE' in event:
self._numBallsCaptured += 1
endOfTrial = True
elif event == 'OUT_OF_TIME':
self._numOutOfTime += 1
endOfTrial = True
elif event == 'HFO_FINISHED':
self._done = True
if endOfTrial:
self._numTrials += 1
print 'EndOfTrial: %d / %d %d %s'%\
(self._numGoals, self._numTrials, self._frame, event)
self._numFrames += self._frame - self._lastTrialStart
self._lastTrialStart = self._frame
def _hear(self, body):
""" Handle a hear message. """
if body[0] == 'referee':
self._hearRef(body)
return
timestep,playerInfo,msg = body
try:
_,team,player = playerInfo[:3]
length = int(msg[0])
except:
return
msg = msg[1:length+1]
if msg == 'START':
if self._isPlaying:
print 'Already playing, ignoring message'
else:
self.startGame()
elif msg == 'DONE':
raise DoneError
elif msg == 'ready':
print 'Agent Connected:', team, player
self._agentReady.add((team, player))
else:
print 'Unhandled message from agent: %s' % msg
def initMsgHandlers(self):
""" Create handlers for different messages. """
self._msgHandlers = []
self.ignoreMsg('player_param')
self.ignoreMsg('player_type')
self.ignoreMsg('ok','change_mode')
self.ignoreMsg('ok','ear')
self.ignoreMsg('ok','move')
self.ignoreMsg('ok','recover')
self.ignoreMsg('ok','say')
self.ignoreMsg('server_param')
self.registerMsgHandler(self._hear,'hear')
def recv(self, retryCount=None):
""" Recieve a message. Retry a specified number of times. """
return self._comm.recvMsg(retryCount=retryCount).strip()
def send(self, msg):
""" Send a message. """
self._comm.sendMsg(msg)
def checkMsg(self, expectedMsg, retryCount=None):
""" Check that the next message is same as expected message. """
msg = self.recv(retryCount)
if msg != expectedMsg:
print >>sys.stderr,'Error with message'
print >>sys.stderr,' expected: %s' % expectedMsg
print >>sys.stderr,' received: %s' % msg
# print >>sys.stderr,len(expectedMsg),len(msg)
raise ValueError
def convertToExtPlayer(self, team, num):
""" Returns the external player number for a given player. """
assert team == self._offenseTeamName or team == self._defenseTeamName,\
'Invalid team name %s. Valid choices: %s, %s.'\
%(team, self._offenseTeamName, self._defenseTeamName)
if team == self._offenseTeamName:
return self._offenseOrder[num]
else:
return self._defenseOrder[num]
def registerMsgHandler(self,handler,*args,**kwargs):
'''Register a message handler.
Handler will be called on a message that matches *args.
'''
args = list(args)
i,_,_ = self._findHandlerInd(args)
if i < 0:
self._msgHandlers.append([args,handler])
else:
if ('quiet' not in kwargs) or (not kwargs['quiet']):
print 'Updating handler for %s' % (' '.join(args))
self._msgHandlers[i] = [args,handler]
def unregisterMsgHandler(self, *args):
""" Delete a message handler. """
i,_,_ = self._findHandlerInd(args)
assert(i >= 0)
del self._msgHandlers[i]
def _findHandlerInd(self, msg):
""" Find the handler for a particular message. """
msg = list(msg)
for i,(partial,handler) in enumerate(self._msgHandlers):
recPartial = msg[:len(partial)]
if recPartial == partial:
return i,len(partial),handler
return -1,None,None
def handleMsg(self, msg):
""" Handle a message using the registered handlers. """
i,prefixLength,handler = self._findHandlerInd(msg)
if i < 0:
print 'Unhandled message:',msg[0:2]
else:
handler(msg[prefixLength:])
def ignoreMsg(self,*args,**kwargs):
""" Ignore a certain type of message. """
self.registerMsgHandler(lambda x: None,*args,**kwargs)
def listenAndProcess(self, retry_count=None):
""" Gather messages and process them. """
try:
msg = self.recv(retry_count)
assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
msg = self.parseMsg(msg)
self.handleMsg(msg)
except TimeoutError:
pass
def disconnectPlayer(self, player, player_num, on_offense):
"""Wait on a launched player to disconnect from the server. """
# print 'Disconnect %s-%d'%(self._offenseTeamName if on_offense
# else self._defenseTeamName, player_num)
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
self.send('(disconnect_player %s %d)'%(team_name, player_num))
player.kill()
def waitOnPlayer(self, player_num, on_offense):
"""Wait on a launched player to connect and be reported by the
server.
"""
self.send('(look)')
partial = ['ok','look']
self._numPlayers = 0
def f(body):
del self._connectedPlayers[:]
for i in xrange(4, len(body)):
_,team,num = body[i][0][:3]
if (team, num) not in self._connectedPlayers:
self._connectedPlayers.append((team,num))
self.registerMsgHandler(f,*partial,quiet=True)
team_name = self._offenseTeamName if on_offense else self._defenseTeamName
while (team_name, str(player_num)) not in self._connectedPlayers:
self.listenAndProcess()
self.send('(look)')
self.ignoreMsg(*partial,quiet=True)
def checkIfAllPlayersConnected(self):
""" Returns true if all players are connected. """
print 'Checking all players are connected'
self.send('(look)')
partial = ['ok','look']
self._numPlayers = 0
def f(x):
self._numPlayers = len(x) - 4 # -4 for time, ball, goal_l, and goal_r
self.send('(look)')
self.registerMsgHandler(f,*partial,quiet=True)
while self._numPlayers != self._numOffense + self._numDefense:
self.listenAndProcess()
self.ignoreMsg(*partial,quiet=True)
def startGame(self):
""" Starts a game of HFO. """
self.send('(change_mode play_on)')
self._isPlaying = True
def printStats(self):
print 'TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
%(self._numFrames,
self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan'))
print 'Trials : %i' % self._numTrials
print 'Goals : %i' % self._numGoals
print 'Defense Captured : %i' % self._numBallsCaptured
print 'Balls Out of Bounds: %i' % self._numBallsOOB
print 'Out of Time : %i' % self._numOutOfTime
def checkLive(self, necProcesses):
"""Returns true if each of the necessary processes is still alive and
running.
"""
for p,name in necProcesses:
if p.poll() is not None:
print 'Something necessary closed (%s), exiting' % name
return False
return True
def run(self, necProcesses):
""" Run the trainer """
try:
self.setTeams()
offense_unums = self._offenseOrder[1: self._numOffense + 1]
sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
defense_unums = self._defenseOrder[: self._numDefense]
sorted_defense_agent_unums = sorted(self._defenseOrder[:self._defenseAgents])
# Launch offense
agent_num = 0
for player_num in xrange(1, 12):
if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num
agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
play_offense=True, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
agent_num += 1
else:
player = self.launch_npc(player_num, play_offense=True)
if player_num in offense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'offense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=True)
# Launch defense
agent_num = 0
for player_num in xrange(1, 12):
if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
port = self._agentServerPort + agent_num + self._offenseAgents
agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
play_offense=False, port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
agent_num += 1
else:
player = self.launch_npc(player_num, play_offense=False)
if player_num in defense_unums:
self._npcPopen.append(player)
necProcesses.append([player, 'defense_npc_' + str(player_num)])
else:
self.disconnectPlayer(player, player_num, on_offense=False)
self.checkIfAllPlayersConnected()
if self._numAgents > 0:
print 'Agents awaiting your connections'
necOff = set([(self._offenseTeamName,str(x)) for x in sorted_offense_agent_unums])
necDef = set([(self._defenseTeamName,str(x)) for x in sorted_defense_agent_unums])
necAgents = necOff.union(necDef)
while self.checkLive(necProcesses) and self._agentReady != necAgents:
self.listenAndProcess()
# Broadcast the HFO configuration
offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
for i in xrange(1, self._numOffense + 1)])
defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
for i in xrange(self._numDefense)])
self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
' num_defense %d offense_nums %s defense_nums %s)'
%(self._offenseTeamName, self._defenseTeamName,
self._numOffense, self._numDefense,
offense_nums, defense_nums))
print 'Starting game'
self.startGame()
while self.checkLive(necProcesses) and not self._done:
prevFrame = self._frame
self.listenAndProcess()
except TimeoutError:
print 'Haven\'t heard from the server for too long, Exiting'
except (KeyboardInterrupt, DoneError):
print 'Finished'
finally:
try:
self._comm.sendMsg('(bye)')
except:
pass
for p in self._agentPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
for p in self._npcPopen:
try:
p.terminate()
time.sleep(0.1)
p.kill()
except:
pass
self._comm.close()
self.printStats()
class Trainer(object):
""" Trainer is responsible for setting up the players and game.
"""
def __init__(self,
args,
rng=numpy.random.RandomState(),
server_port=6001,
coach_port=6002):
self._rng = rng # The Random Number Generator
self._serverPort = server_port # The port the server is listening on
self._coachPort = coach_port # The coach port to talk with the server
self._logDir = args.logDir # Directory to store logs
self._record = args.record # Record states + actions
self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
self._maxTrials = args.numTrials # Maximum number of trials to play
self._maxFrames = args.numFrames # Maximum number of frames to play
self._maxFramesPerTrial = args.maxFramesPerTrial
# =============== FIELD DIMENSIONS =============== #
self.NUM_FRAMES_TO_HOLD = 2 # Hold ball this many frames to capture
self.HOLD_FACTOR = 1.5 # Gain to calculate ball control
self.PITCH_WIDTH = 68.0 # Width of the field
self.PITCH_LENGTH = 105.0 # Length of field in long-direction
self.UNTOUCHED_LENGTH = 100 # Trial will end if ball untouched for this long
# allowedBallX, allowedBallY defines the usable area of the playfield
self._allowedBallX = numpy.array([-0.1, 0.5 * self.PITCH_LENGTH])
self._allowedBallY = numpy.array(
[-0.5 * self.PITCH_WIDTH, 0.5 * self.PITCH_WIDTH])
# =============== COUNTERS =============== #
self._numFrames = 0 # Number of frames seen in HFO trials
self._frame = 0 # Current frame id
self._lastTrialStart = -1 # Frame Id in which the last trial started
self._lastFrameBallTouched = -1 # Frame Id in which ball was last touched
# =============== TRIAL RESULTS =============== #
self._numTrials = 0 # Total number of HFO trials
self._numGoals = 0 # Trials in which the offense scored a goal
self._numBallsCaptured = 0 # Trials in which defense captured the ball
self._numBallsOOB = 0 # Trials in which ball went out of bounds
self._numOutOfTime = 0 # Trials that ran out of time
# =============== AGENT =============== #
self._numAgents = args.offenseAgents + args.defenseAgents
self._offenseAgents = args.offenseAgents
self._defenseAgents = args.defenseAgents
self._agentTeams = [] # Names of the teams the agents are playing for
self._agentNumInt = [] # List of agents internal team numbers
self._agentNumExt = [] # List of agents external team numbers
self._agentServerPort = args.port # Base Port for agent's server
self._agentOnBall = args.agent_on_ball # If true, agent starts with the ball
# =============== MISC =============== #
self._offenseTeamName = '' # Name of the offensive team
self._defenseTeamName = '' # Name of the defensive team
self._playerPositions = numpy.zeros(
(11, 2, 2)) # Positions of the players
self._ballPosition = numpy.zeros(2) # Position of the ball
self._ballHeld = numpy.zeros((11, 2)) # Track player holding the ball
self._teams = [] # Team indexes for offensive and defensive teams
self._SP = {
} # Sever Parameters. Recieved when connecting to the server.
self._isPlaying = False # Is a game being played?
self._teamHoldingBall = None # Team currently in control of the ball
self._playerHoldingBall = None # Player current in control of ball
self._agentPopen = [] # Agent's processes
self.initMsgHandlers()
def launch_agent(self, agent_num, play_offense, port):
"""Launch the learning agent using the start_agent.sh script and
return a DummyPopen for the process.
"""
print '[Trainer] Launching Agent', str(agent_num)
if play_offense:
assert self._numOffense > 0
team_name = self._offenseTeamName
self._agentTeams.append(team_name)
# First offense number is reserved for inactive offensive goalie
internal_player_num = agent_num + 1
self._agentNumInt.append(internal_player_num)
numTeammates = self._numOffense - 1
numOpponents = self._numDefense
else:
assert self._numDefense > 0
team_name = self._defenseTeamName
self._agentTeams.append(team_name)
internal_player_num = agent_num
self._agentNumInt.append(internal_player_num)
numTeammates = self._numDefense - 1
numOpponents = self._numOffense
ext_num = self.convertToExtPlayer(team_name, internal_player_num)
self._agentNumExt.append(ext_num)
binary_dir = os.path.dirname(os.path.realpath(__file__))
agentCmd = 'start_agent.sh -t %s -u %i -p %i -P %i --log-dir %s'\
' --numTeammates %i --numOpponents %i'\
' --playingOffense %i --serverPort %i'\
%(team_name, ext_num, self._serverPort,
self._coachPort, self._logDir, numTeammates,
numOpponents, play_offense, port)
if self._record:
agentCmd += ' --record'
agentCmd = os.path.join(binary_dir, agentCmd)
agentCmd = agentCmd.split(' ')
# Ignore stderr because librcsc continually prints to it
kwargs = {} #{'stderr':open('/dev/null','w')}
p = subprocess.Popen(agentCmd, **kwargs)
p.wait()
pid_file = os.path.join(self._logDir, 'start%i' % p.pid)
print '[Trainer] Parsing agent\'s pid from file:', pid_file
assert os.path.isfile(pid_file)
with open(pid_file, 'r') as f:
output = f.read()
pid = int(re.findall('PID: (\d+)', output)[0])
return DummyPopen(pid)
def getDefensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer defense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [9, 10, 8, 11, 7, 4, 6, 2, 3, 5]
elif team_name == 'WrightEagle':
return [5, 2, 8, 9, 10, 6, 3, 11, 4, 7]
else:
return [2, 3, 4, 5, 6, 7, 8, 11, 9, 10]
def getOffensiveRoster(self, team_name):
"""Returns a list of player numbers on a given team that are thought
to prefer offense. This map is not set in stone as the players on
some teams can adapt and change their roles.
"""
if team_name == 'Borregos':
return [2, 4, 6, 5, 3, 7, 9, 10, 8, 11]
elif team_name == 'WrightEagle':
return [11, 4, 7, 3, 6, 10, 8, 9, 2, 5]
else:
return [11, 7, 8, 9, 10, 6, 3, 2, 4, 5]
def setTeams(self):
""" Sets the offensive and defensive teams and player rosters. """
self._offenseTeamInd = 0
self._offenseTeamName = self._teams[self._offenseTeamInd]
self._defenseTeamName = self._teams[1 - self._offenseTeamInd]
offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
self._offenseOrder = [1] + offensive_roster # 1 for goalie
self._defenseOrder = [1] + defensive_roster # 1 for goalie
def teamToInd(self, team_name):
""" Returns the index of a given team. """
return self._teams.index(team_name)
def parseMsg(self, msg):
""" Parse a message """
assert (msg[0] == '(')
res, ind = self.__parseMsg(msg, 1)
assert (ind == len(msg)), msg
return res
def __parseMsg(self, msg, ind):
""" Recursively parse a message. """
res = []
while True:
if msg[ind] == '"':
ind += 1
startInd = ind
while msg[ind] != '"':
ind += 1
res.append(msg[startInd:ind])
ind += 1
elif msg[ind] == '(':
inner, ind = self.__parseMsg(msg, ind + 1)
res.append(inner)
elif msg[ind] == ' ':
ind += 1
elif msg[ind] == ')':
return res, ind + 1
else:
startInd = ind
while msg[ind] not in ' ()':
ind += 1
res.append(msg[startInd:ind])
def initComm(self):
""" Initialize communication to server. """
self._comm = ClientCommunicator(port=self._coachPort)
self.send('(init (version 8.0))')
self.checkMsg('(init ok)', retryCount=5)
# self.send('(eye on)')
self.send('(ear on)')
def _hear(self, body):
""" Handle a hear message. """
timestep, playerInfo, msg = body
if len(playerInfo) != 3:
return
_, team, player = playerInfo
try:
length = int(msg[0])
except:
return
msg = msg[1:length + 1]
if msg == 'START':
if self._isPlaying:
print '[Trainer] Already playing, ignoring message'
else:
self.startGame()
elif msg == 'DONE':
raise DoneError
else:
print '[Trainer] Unhandled message from agent: %s' % msg
def initMsgHandlers(self):
""" Create handlers for different messages. """
self._msgHandlers = []
self.ignoreMsg('player_param')
self.ignoreMsg('player_type')
self.ignoreMsg('ok', 'change_mode')
self.ignoreMsg('ok', 'ear')
self.ignoreMsg('ok', 'move')
self.ignoreMsg('ok', 'recover')
self.ignoreMsg('ok', 'say')
self.registerMsgHandler(self._handleSP, 'server_param')
self.registerMsgHandler(self._hear, 'hear')
def recv(self, retryCount=None):
""" Recieve a message. Retry a specified number of times. """
return self._comm.recvMsg(retryCount=retryCount).strip()
def send(self, msg):
""" Send a message. """
self._comm.sendMsg(msg)
def checkMsg(self, expectedMsg, retryCount=None):
""" Check that the next message is same as expected message. """
msg = self.recv(retryCount)
if msg != expectedMsg:
print >> sys.stderr, '[Trainer] Error with message'
print >> sys.stderr, ' expected: %s' % expectedMsg
print >> sys.stderr, ' received: %s' % msg
# print >>sys.stderr,len(expectedMsg),len(msg)
raise ValueError
def extractPoint(self, msg):
""" Extract a point from the provided message. """
return numpy.array(map(float, msg[:2]))
def convertToExtPlayer(self, team, num):
""" Returns the external player number for a given player. """
assert team == self._offenseTeamName or team == self._defenseTeamName,\
'Invalid team name %s. Valid choices: %s, %s.'\
%(team, self._offenseTeamName, self._defenseTeamName)
if team == self._offenseTeamName:
return self._offenseOrder[num]
else:
return self._defenseOrder[num]
def convertFromExtPlayer(self, team, num):
""" Maps external player number to internal player number. """
if team == self._offenseTeamName:
return self._offenseOrder.index(num)
else:
return self._defenseOrder.index(num)
def seeGlobal(self, body):
"""Send a look message to extract global information on ball and
player positions.
"""
self.send('(look)')
self._frame = int(body[0])
for obj in body[1:]:
objType = obj[0]
objData = obj[1:]
if objType[0] == 'g':
continue
elif objType[0] == 'b':
self._ballPosition = self.extractPoint(objData)
elif objType[0] == 'p':
teamName = objType[1]
team = self.teamToInd(teamName)
playerNum = self.convertFromExtPlayer(teamName,
int(objType[2]))
self._playerPositions[playerNum, :,
team] = self.extractPoint(objData)
def registerMsgHandler(self, handler, *args, **kwargs):
'''Register a message handler.
Handler will be called on a message that matches *args.
'''
args = list(args)
i, _, _ = self._findHandlerInd(args)
if i < 0:
self._msgHandlers.append([args, handler])
else:
if ('quiet' not in kwargs) or (not kwargs['quiet']):
print '[Trainer] Updating handler for %s' % (' '.join(args))
self._msgHandlers[i] = [args, handler]
def unregisterMsgHandler(self, *args):
""" Delete a message handler. """
i, _, _ = self._findHandlerInd(args)
assert (i >= 0)
del self._msgHandlers[i]
def _findHandlerInd(self, msg):
""" Find the handler for a particular message. """
msg = list(msg)
for i, (partial, handler) in enumerate(self._msgHandlers):
recPartial = msg[:len(partial)]
if recPartial == partial:
return i, len(partial), handler
return -1, None, None
def handleMsg(self, msg):
""" Handle a message using the registered handlers. """
i, prefixLength, handler = self._findHandlerInd(msg)
if i < 0:
print '[Trainer] Unhandled message:', msg[0:2]
else:
handler(msg[prefixLength:])
def ignoreMsg(self, *args, **kwargs):
""" Ignore a certain type of message. """
self.registerMsgHandler(lambda x: None, *args, **kwargs)
def _handleSP(self, body):
""" Handler for the sever params message. """
for param in body:
try:
val = int(param[1])
except:
try:
val = float(param[1])
except:
val = param[1]
self._SP[param[0]] = val
def listenAndProcess(self):
""" Gather messages and process them. """
msg = self.recv()
assert ((msg[0] == '(') and (msg[-1] == ')')), '|%s|' % msg
msg = self.parseMsg(msg)
self.handleMsg(msg)
def _readTeamNames(self, body):
""" Read the names of each of the teams. """
self._teams = []
for _, _, team in body:
self._teams.append(team)
time.sleep(0.1)
self.send('(team_names)')
def waitOnTeam(self, first):
"""Wait on a given team. First indicates if this is the first team
connected or the second.
"""
self.send('(team_names)')
partial = ['ok', 'team_names']
self.registerMsgHandler(self._readTeamNames, *partial, quiet=True)
while len(self._teams) < (1 if first else 2):
self.listenAndProcess()
#self.unregisterMsgHandler(*partial)
self.ignoreMsg(*partial, quiet=True)
def checkIfAllPlayersConnected(self):
""" Returns true if all players are connected. """
self.send('(look)')
partial = ['ok', 'look']
self._numPlayers = 0
def f(x):
self._numPlayers = len(
x) - 4 # -4 for time, ball, goal_l, and goal_r
self.send('(look)')
self.registerMsgHandler(f, *partial)
while self._numPlayers != 2 * 11:
self.listenAndProcess()
self.ignoreMsg(*partial, quiet=True)
def startGame(self):
""" Starts a game of HFO. """
self.reset()
self.registerMsgHandler(self.seeGlobal, 'see_global')
self.registerMsgHandler(self.seeGlobal, 'ok', 'look', quiet=True)
#self.registerMsgHandler(self.checkBall,'ok','check_ball')
self.send('(look)')
self._isPlaying = True
def calcBallHolder(self):
'''Calculates the ball holder, returns results in teamInd, playerInd. '''
totalHeld = 0
for team in self._teams:
for i in range(11):
pos = self._playerPositions[i, :, self.teamToInd(team)]
distBound = self._SP['kickable_margin'] + self._SP['player_size'] \
+ self._SP['ball_size']
distBound *= self.HOLD_FACTOR
if numpy.linalg.norm(self._ballPosition - pos) < distBound:
self._ballHeld[i, self.teamToInd(team)] += 1
totalHeld += 1
else:
self._ballHeld[i, self.teamToInd(team)] = 0
# If multiple players are close to the ball, no-one is holding
if totalHeld > 1:
self._ballHeld[:, :] = 0
inds = numpy.transpose(
(self._ballHeld >= self.NUM_FRAMES_TO_HOLD).nonzero())
assert (len(inds) <= 1)
if len(inds) == 1:
return inds[0, 1], inds[0, 0]
else:
return None, None
def isGoal(self):
""" Returns true if a goal has been scored. """
return (self._ballPosition[0] > self._allowedBallX[1]) \
and (numpy.abs(self._ballPosition[1]) <= 0.5 * self._SP['goal_width'])
def isOOB(self):
""" Returns true if the ball is out of bounds. """
return self._ballPosition[0] < self._allowedBallX[0] \
or self._ballPosition[0] > self._allowedBallX[1] \
or self._ballPosition[1] < self._allowedBallY[0] \
or self._ballPosition[1] > self._allowedBallY[1]
def isCaptured(self):
""" Returns true if the ball is captured by defense. """
return self._teamHoldingBall not in [None, self._offenseTeamInd]
def isOOT(self):
""" Returns true if the trial has run out of time. """
return self._frame - self._lastFrameBallTouched > self.UNTOUCHED_LENGTH \
or (self._maxFramesPerTrial > 0 and self._frame -
self._lastTrialStart > self._maxFramesPerTrial)
def movePlayer(self, team, internal_num, pos, convertToExt=True):
""" Move a player to a specified position.
Args:
team: the team name of the player
interal_num: the player's internal number
pos: position to move player to
convertToExt: convert interal player num to external
"""
num = self.convertToExtPlayer(team, internal_num) if convertToExt \
else internal_num
self.send('(move (player %s %i) %f %f)' % (team, num, pos[0], pos[1]))
def moveBall(self, pos):
""" Moves the ball to a specified x,y position. """
self.send('(move (ball) %f %f 0.0 0.0 0.0)' % tuple(pos))
def randomPointInBounds(self, xBounds=None, yBounds=None):
"""Returns a random point inside of the box defined by xBounds,
yBounds. Where xBounds=[x_min, x_max] and yBounds=[y_min,
y_max]. Defaults to the xy-bounds of the playable HFO area.
"""
if xBounds is None:
xBounds = self.allowedBallX
if yBounds is None:
yBounds = self.allowedBallY
pos = numpy.zeros(2)
bounds = [xBounds, yBounds]
for i in range(2):
pos[i] = self._rng.rand() * (bounds[i][1] -
bounds[i][0]) + bounds[i][0]
return pos
def boundPoint(self, pos):
"""Ensures a point is within the minimum and maximum bounds of the
HFO playing area.
"""
pos[0] = min(max(pos[0], self._allowedBallX[0]), self._allowedBallX[1])
pos[1] = min(max(pos[1], self._allowedBallY[0]), self._allowedBallY[1])
return pos
def reset(self):
""" Resets the HFO domain by moving the ball and players. """
self.resetBallPosition()
self.resetPlayerPositions()
self.send('(recover)')
self.send('(change_mode play_on)')
# self.send('(say RESET)')
def resetBallPosition(self):
"""Reset the position of the ball for a new HFO trial. """
self._ballPosition = self.boundPoint(
self.randomPointInBounds(
.2 * self._allowedBallX + .05 * self.PITCH_LENGTH,
.8 * self._allowedBallY))
self.moveBall(self._ballPosition)
def getOffensiveResetPosition(self):
""" Returns a random position for an offensive player. """
offsets = [
[-1, -1],
[-1, 1],
[1, 1],
[1, -1],
[0, 2],
[0, -2],
[-2, -2],
[-2, 2],
[2, 2],
[2, -2],
]
offset = offsets[self._rng.randint(len(offsets))]
offset_from_ball = 0.1 * self.PITCH_LENGTH * self._rng.rand(2) + \
0.1 * self.PITCH_LENGTH * numpy.array(offset)
return self.boundPoint(self._ballPosition + offset_from_ball)
# return self._ballPosition
def getDefensiveResetPosition(self):
""" Returns a random position for a defensive player. """
return self.boundPoint(
self.randomPointInBounds([
0.5 * 0.5 * self.PITCH_LENGTH, 0.75 * 0.5 * self.PITCH_LENGTH
], 0.8 * self._allowedBallY))
def resetPlayerPositions(self):
"""Reset the positions of the players. This is called after a trial
ends to setup for the next trial.
"""
# Always Move the offensive goalie to the left goal
self.movePlayer(self._offenseTeamName, 0,
[-0.5 * self.PITCH_LENGTH, 0])
# Move the rest of the offense
for i in xrange(1, self._numOffense + 1):
self.movePlayer(self._offenseTeamName, i,
self.getOffensiveResetPosition())
# Move the agent to the ball
if self._agentOnBall and self._offenseAgents > 0:
self.movePlayer(self._offenseTeamName, self._agentNumInt[0],
self._ballPosition)
# Move the defensive goalie
if self._numDefense > 0:
self.movePlayer(self._defenseTeamName, 0,
[0.5 * self.PITCH_LENGTH, 0])
# Move the rest of the defense
for i in xrange(1, self._numDefense):
self.movePlayer(self._defenseTeamName, i,
self.getDefensiveResetPosition())
def removeNonHFOPlayers(self):
"""Removes players that aren't involved in HFO game.
The players whose numbers are greater than numOffense/numDefense
are sent to left-field.
"""
offensive_agent_numbers = self._agentNumInt[:self._offenseAgents]
defensive_agent_numbers = self._agentNumInt[self._offenseAgents:]
for i in xrange(self._numOffense + 1, 11):
if i not in offensive_agent_numbers:
self.movePlayer(self._offenseTeamName, i,
[-0.25 * self.PITCH_LENGTH, 0])
for i in xrange(self._numDefense, 11):
if i not in defensive_agent_numbers:
self.movePlayer(self._defenseTeamName, i,
[-0.25 * self.PITCH_LENGTH, 0])
def step(self):
""" Takes a simulated step. """
# self.send('(check_ball)')
self.removeNonHFOPlayers()
self._teamHoldingBall, self._playerHoldingBall = self.calcBallHolder()
if self._teamHoldingBall is not None:
self._lastFrameBallTouched = self._frame
if self.trialOver():
self.updateResults()
self._lastFrameBallTouched = self._frame
self.reset()
def updateResults(self):
""" Updates the various members after a trial has ended. """
if self.isGoal():
self._numGoals += 1
result = 'Goal'
self.send('(say GOAL)')
elif self.isOOB():
self._numBallsOOB += 1
result = 'Out of Bounds'
self.send('(say OUT_OF_BOUNDS)')
elif self.isCaptured():
self._numBallsCaptured += 1
result = 'Defense Captured'
self.send('(say CAPTURED_BY_DEFENSE)')
elif self.isOOT():
self._numOutOfTime += 1
result = 'Ball untouched for too long'
self.send('(say OUT_OF_TIME)')
else:
print '[Trainer] Error: Unable to detect reason for End of Trial!'
sys.exit(1)
print '[Trainer] EndOfTrial: %2i /%2i %5i %s'%\
(self._numGoals, self._numTrials, self._frame, result)
self._numTrials += 1
self._numFrames += self._frame - self._lastTrialStart
self._lastTrialStart = self._frame
if (self._maxTrials > 0) and (self._numTrials >= self._maxTrials):
raise DoneError
if (self._maxFrames > 0) and (self._numFrames >= self._maxFrames):
raise DoneError
def trialOver(self):
"""Returns true if the trial has ended for one of the following
reasons: Goal scored, out of bounds, captured by defense, or
untouched for too long.
"""
# The trial is still being setup, it cannot be over.
if self._frame - self._lastTrialStart < 5:
return False
return self.isGoal() or self.isOOB() or self.isCaptured(
) or self.isOOT()
def printStats(self):
print 'Num frames in completed trials : %i' % (self._numFrames)
print 'Trials : %i' % self._numTrials
print 'Goals : %i' % self._numGoals
print 'Defense Captured : %i' % self._numBallsCaptured
print 'Balls Out of Bounds: %i' % self._numBallsOOB
print 'Out of Time : %i' % self._numOutOfTime
def checkLive(self, necProcesses):
"""Returns true if each of the necessary processes is still alive and
running.
"""
for p, name in necProcesses:
if p.poll() is not None:
print '[Trainer] Something necessary closed (%s), exiting' % name
return False
return True
def run(self, necProcesses):
""" Run the trainer """
try:
for agent_num in xrange(self._offenseAgents):
port = self._agentServerPort + agent_num
agent = self.launch_agent(agent_num,
play_offense=True,
port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
for agent_num in xrange(self._defenseAgents):
port = self._agentServerPort + agent_num + self._offenseAgents
agent = self.launch_agent(agent_num,
play_offense=False,
port=port)
self._agentPopen.append(agent)
necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
# Broadcast the HFO configuration
offense_nums = ' '.join([
str(self.convertToExtPlayer(self._offenseTeamName, i))
for i in xrange(1, self._numOffense + 1)
])
defense_nums = ' '.join([
str(self.convertToExtPlayer(self._defenseTeamName, i))
for i in xrange(self._numDefense)
])
self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
' num_defense %d offense_nums %s defense_nums %s)'
%(self._offenseTeamName, self._defenseTeamName,
self._numOffense, self._numDefense,
offense_nums, defense_nums))
self.startGame()
while self.checkLive(necProcesses):
prevFrame = self._frame
self.listenAndProcess()
if self._frame != prevFrame:
self.step()
except TimeoutError:
print '[Trainer] Haven\'t heard from the server for too long, Exiting'
except (KeyboardInterrupt, DoneError):
print '[Trainer] Exiting'
finally:
for p in self._agentPopen:
p.send_signal(SIGINT)
try:
self._comm.sendMsg('(bye)')
except:
pass
self._comm.close()
self.printStats()