Exemplo n.º 1
0
class Trainer(object):
  """ Trainer is responsible for setting up the players and game.
  """
  def __init__(self, args, server_port=6001, coach_port=6002):
    self._serverPort = server_port  # The port the server is listening on
    self._coachPort = coach_port # The coach port to talk with the server
    self._logDir = args.logDir # Directory to store logs
    self._record = args.record # Record states + actions
    self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
    self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
    # =============== COUNTERS =============== #
    self._numFrames = 0 # Number of frames seen in HFO trials
    self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
    self._frame = 0 # Current frame id
    self._lastTrialStart = -1 # Frame Id in which the last trial started
    # =============== TRIAL RESULTS =============== #
    self._numTrials = 0 # Total number of HFO trials
    self._numGoals = 0 # Trials in which the offense scored a goal
    self._numBallsCaptured = 0 # Trials in which defense captured the ball
    self._numBallsOOB = 0 # Trials in which ball went out of bounds
    self._numOutOfTime = 0 # Trials that ran out of time
    # =============== AGENT =============== #
    self._numAgents = args.offenseAgents + args.defenseAgents
    self._offenseAgents = args.offenseAgents
    self._defenseAgents = args.defenseAgents
    self._agentPlayGoalie = args.agentPlayGoalie
    self._agentReady = set([]) # Unums of ready agents
    self._agentTeams = [] # Names of the teams the agents are playing for
    self._agentNumInt = [] # List of agents internal team numbers
    self._agentServerPort = args.port # Base Port for agent's server
    # =============== MISC =============== #
    self._offenseTeamName = '' # Name of the offensive team
    self._defenseTeamName = '' # Name of the defensive team
    self._isPlaying = False # Is a game being played?
    self._done = False # Are we finished?
    self._agentPopen = [] # Agent's processes
    self._npcPopen = [] # NPC's processes
    self._connectedPlayers = [] # List of connected players
    self.initMsgHandlers()

  def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
    """Launches a learning agent using the agent binary

    Returns a Popen process object
    """
    if play_offense:
      assert self._numOffense > 0
      team_name = self._offenseTeamName
      self._agentTeams.append(team_name)
      # First offense number is reserved for inactive offensive goalie
      internal_player_num = agent_num + 1
      self._agentNumInt.append(internal_player_num)
    else:
      assert self._numDefense > 0
      team_name = self._defenseTeamName
      self._agentTeams.append(team_name)
      internal_player_num = agent_num
      self._agentNumInt.append(internal_player_num)
    binary_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                              'teams', 'base')
    config_dir = os.path.join(binary_dir, 'config/formations-dt')
    print(("Waiting for player-controlled agent %s-%d: config_dir=%s, "\
          "server_port=%d, server_addr=%s, team_name=%s, play_goalie=%r"
          % (self._offenseTeamName if play_offense else self._defenseTeamName,
             agent_num, config_dir, self._serverPort, "localhost", team_name,
             agent_ext_num==1)))
    if wait_until_join:
      self.waitOnPlayer(agent_ext_num, play_offense)
    return None

  def createTeam(self, requested_team_name, play_offense):
    """ Given a team name, returns the team object. """
    teams_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'teams')
    if requested_team_name == 'helios':
      print('Creating team Helios')
      team_name = 'HELIOS_' + ('left' if play_offense else 'right')
      team_dir = os.path.join(teams_dir, 'helios', 'helios-13Eindhoven')
      lib_dir = os.path.join(teams_dir, 'helios', 'local', 'lib')
      return Teams.Helios(team_name, team_dir, lib_dir,
                          binaryName='helios_player', host='localhost',
                          port=self._serverPort)
    elif requested_team_name == 'base':
      print('Creating team Agent2d (base)')
      team_name = 'base_' + ('left' if play_offense else 'right')
      team_dir = os.path.join(teams_dir, 'base')
      lib_dir = None
      return Teams.Agent2d(team_name, team_dir, lib_dir,
                           binaryName='sample_player', logDir=self._logDir,
                           record=self._record, host='localhost',
                           port=self._serverPort)
    elif requested_team_name == 'robocin':
      print('Creating team RoboCIN2d')
      team_name = 'RoboCIn_' + ('left' if play_offense else 'right')
      team_dir = os.path.join(teams_dir, 'robocin', 'src')
      lib_dir = None
      return Teams.RoboCIn(team_name, team_dir, lib_dir,
                           binaryName='robocin_player', logDir=self._logDir,
                           record=self._record, host='localhost',
                           port=self._serverPort)
    else:
      print('Unknown team requested: ' + requested_team_name)
      sys.exit(1)

  def getTeams(self, offense_team_name, defense_team_name):
    """ Sets the offensive and defensive teams and player rosters. """
    # Set up offense team
    offenseTeam = self.createTeam(offense_team_name, play_offense=True)
    self._offenseTeamName = offenseTeam._name
    self._offenseOrder = [1] + offenseTeam._offense_order # 1 for goalie
    # Set up defense team
    defenseTeam = self.createTeam(defense_team_name, play_offense=False)
    self._defenseTeamName = defenseTeam._name
    self._defenseOrder = [1] + defenseTeam._defense_order # 1 for goalie
    return (offenseTeam, defenseTeam)

  def parseMsg(self, msg):
    """ Parse a message """
    assert(msg[0] == '(')
    res, ind = self.__parseMsg(msg,1)
    assert(ind == len(msg)), msg
    return res

  def __parseMsg(self, msg, ind):
    """ Recursively parse a message. """
    res = []
    while True:
      if msg[ind] == '"':
        ind += 1
        startInd = ind
        while msg[ind] != '"':
          ind += 1
        res.append(msg[startInd:ind])
        ind += 1
      elif msg[ind] == '(':
        inner,ind = self.__parseMsg(msg,ind+1)
        res.append(inner)
      elif msg[ind] == ' ':
        ind += 1
      elif msg[ind] == ')':
        return res,ind+1
      else:
        startInd = ind
        while msg[ind] not in ' ()':
          ind += 1
        res.append(msg[startInd:ind])

  def initComm(self):
    """ Initialize communication to server. """
    self._comm = ClientCommunicator(port=self._coachPort)
    self.send('(init (version 8.0))')
    self.checkMsg('(init ok)', retryCount=5)
    # self.send('(eye on)')
    self.send('(ear on)')

  def _hearRef(self, body):
    """ Handles hear messages from referee. """
    assert body[0] == 'referee', 'Expected referee message.'
    _,ts,event = body
    self._frame = int(ts)
    endOfTrial = False
    if 'GOAL' in event:
      self._numGoals += 1
      self._numGoalFrames += self._frame - self._lastTrialStart
      endOfTrial = True
    elif event == 'OUT_OF_BOUNDS':
      self._numBallsOOB += 1
      endOfTrial = True
    elif 'CAPTURED_BY_DEFENSE' in event:
      self._numBallsCaptured += 1
      endOfTrial = True
    elif event == 'OUT_OF_TIME':
      self._numOutOfTime += 1
      endOfTrial = True
    elif event == 'HFO_FINISHED':
      self._done = True
    if endOfTrial:
      self._numTrials += 1
      print('EndOfTrial: %d / %d %d %s'%\
        (self._numGoals, self._numTrials, self._frame, event))
      self._numFrames += self._frame - self._lastTrialStart
      self._lastTrialStart = self._frame
      self.getConnectedPlayers()

  def _hear(self, body):
    """ Handle a hear message. """
    if body[0] == 'referee':
      self._hearRef(body)
      return
    timestep,playerInfo,msg = body
    try:
      _,team,player = playerInfo[:3]
      length = int(msg[0])
    except:
      return
    msg = msg[1:length+1]
    if msg == 'START':
      if self._isPlaying:
        print('Already playing, ignoring message')
      else:
        self.startGame()
    elif msg == 'DONE':
      raise DoneError
    elif msg == 'ready':
      print('Agent Connected:', team, player)
      self._agentReady.add((team, player))
    else:
      print('Unhandled message from agent: %s' % msg)

  def initMsgHandlers(self):
    """ Create handlers for different messages. """
    self._msgHandlers = []
    self.ignoreMsg('player_param')
    self.ignoreMsg('player_type')
    self.ignoreMsg('ok','change_mode')
    self.ignoreMsg('ok','ear')
    self.ignoreMsg('ok','move')
    self.ignoreMsg('ok','recover')
    self.ignoreMsg('ok','say')
    self.ignoreMsg('server_param')
    self.registerMsgHandler(self._hear,'hear')

  def recv(self, retryCount=None):
    """ Recieve a message. Retry a specified number of times. """
    return self._comm.recvMsg(retryCount=retryCount).strip()

  def send(self, msg):
    """ Send a message. """
    self._comm.sendMsg(msg)

  def checkMsg(self, expectedMsg, retryCount=None):
    """ Check that the next message is same as expected message. """
    msg = self.recv(retryCount)
    if msg != expectedMsg:
      sys.stderr.write('Error with message')
      sys.stderr.write('  expected: ' + expectedMsg)
      sys.stderr.write('  received: ' + msg)
      # print >>sys.stderr,len(expectedMsg),len(msg)
      raise ValueError

  def convertToExtPlayer(self, team, num):
    """ Returns the external player number for a given player. """
    assert team == self._offenseTeamName or team == self._defenseTeamName,\
      'Invalid team name %s. Valid choices: %s, %s.'\
      %(team, self._offenseTeamName, self._defenseTeamName)
    if team == self._offenseTeamName:
      return self._offenseOrder[num]
    else:
      return self._defenseOrder[num]

  def registerMsgHandler(self,handler,*args,**kwargs):
    '''Register a message handler.

    Handler will be called on a message that matches *args.

    '''
    args = list(args)
    i,_,_ = self._findHandlerInd(args)
    if i < 0:
      self._msgHandlers.append([args,handler])
    else:
      if ('quiet' not in kwargs) or (not kwargs['quiet']):
        print('Updating handler for %s' % (' '.join(args)))
      self._msgHandlers[i] = [args,handler]

  def unregisterMsgHandler(self, *args):
    """ Delete a message handler. """
    i,_,_ = self._findHandlerInd(args)
    assert(i >= 0)
    del self._msgHandlers[i]

  def _findHandlerInd(self, msg):
    """ Find the handler for a particular message. """
    msg = list(msg)
    for i,(partial,handler) in enumerate(self._msgHandlers):
      recPartial = msg[:len(partial)]
      if recPartial == partial:
        return i,len(partial),handler
    return -1,None,None

  def handleMsg(self, msg):
    """ Handle a message using the registered handlers. """
    i,prefixLength,handler = self._findHandlerInd(msg)
    if i < 0:
      print('Unhandled message:',msg[0:2])
    else:
      handler(msg[prefixLength:])

  def ignoreMsg(self,*args,**kwargs):
    """ Ignore a certain type of message. """
    self.registerMsgHandler(lambda x: None,*args,**kwargs)

  def listenAndProcess(self, retry_count=None):
    """ Gather messages and process them. """
    try:
      msg = self.recv(retry_count)
      assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
      msg = self.parseMsg(msg)
      self.handleMsg(msg)
    except TimeoutError:
      pass

  def disconnectPlayer(self, player, player_num, on_offense):
    """Wait on a launched player to disconnect from the server. """
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    while (team_name, str(player_num)) in self._connectedPlayers:
      self.send('(disconnect_player %s %d)'%(team_name, player_num))
      self.getConnectedPlayers()
    player.kill()

  def getConnectedPlayers(self):
    """ Get the list of connected players. Populates self._connectedPlayers. """
    self._gotLook = False
    self.send('(look)')
    partial = ['ok','look']
    def f(body):
      self._gotLook = True
      del self._connectedPlayers[:]
      for i in range(4, len(body)):
        _,team,num = body[i][0][:3]
        if (team, num) not in self._connectedPlayers:
          self._connectedPlayers.append((team,num))
    self.registerMsgHandler(f,*partial,quiet=True)
    while not self._gotLook:
      self.listenAndProcess()
      self.send('(look)')
    self.ignoreMsg(*partial,quiet=True)

  def waitOnPlayer(self, player_num, on_offense):
    """ Wait on a launched player to connect and be reported by the server. """
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    while (team_name, str(player_num)) not in self._connectedPlayers:
      self.getConnectedPlayers()

  def allPlayersConnected(self):
    """ Returns true all players are connected. """
    return len(self._connectedPlayers) == self._numOffense + self._numDefense

  def sendHFOConfig(self):
    """ Broadcast the HFO configuration """
    offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
                             for i in range(1, self._numOffense + 1)])
    defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
                             for i in range(self._numDefense)])
    self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
                ' num_defense %d offense_nums %s defense_nums %s)'
              %(self._offenseTeamName, self._defenseTeamName,
                self._numOffense, self._numDefense,
                offense_nums, defense_nums))

  def startGame(self):
    """ Starts a game of HFO. """
    self.send('(change_mode play_on)')
    self._isPlaying = True

  def printStats(self):
    print('TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
      %(self._numFrames,
        self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
        self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan')))
    print('Trials             : %i' % self._numTrials)
    print('Goals              : %i' % self._numGoals)
    print('Defense Captured   : %i' % self._numBallsCaptured)
    print('Balls Out of Bounds: %i' % self._numBallsOOB)
    print('Out of Time        : %i' % self._numOutOfTime)

  def checkLive(self, necProcesses):
    """Returns true if each of the necessary processes is still alive and
    running.

    """
    for p,name in necProcesses:
      if p is not None and p.poll() is not None:
        print('Something necessary closed (%s), exiting' % name)
        return False
    return True

  def run(self, necProcesses, offense_team_name, defense_team_name):
    """ Run the trainer """
    try:
      (offenseTeam, defenseTeam) = self.getTeams(offense_team_name, defense_team_name)
      offense_unums = self._offenseOrder[1: self._numOffense + 1]
      sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
      defense_unums = sorted(self._defenseOrder[: self._numDefense])
      sorted_defense_agent_unums = \
        defense_unums[:self._defenseAgents] if self._agentPlayGoalie \
        else defense_unums[-self._defenseAgents:]

      # Launch offense
      agent_num = 0
      for player_num in range(1, 12):
        if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num
          agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
                                    play_offense=True, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = offenseTeam.launch_npc(player_num)
          self.waitOnPlayer(player_num, True)
          if player_num in offense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'offense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=True)

      # Launch defense
      agent_num = 0
      for player_num in range(1, 12):
        if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num + self._offenseAgents
          agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
                                    play_offense=False, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = defenseTeam.launch_npc(player_num)
          self.waitOnPlayer(player_num, False)
          if player_num in defense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'defense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=False)

      print('Checking all players connected')
      while not self.allPlayersConnected():
        self.getConnectedPlayers()

      time.sleep(0.1)
      self.sendHFOConfig()

      print('Starting game')
      time.sleep(0.1)
      self.startGame()
      while self.allPlayersConnected() and self.checkLive(necProcesses) and not self._done:
        prevFrame = self._frame
        self.listenAndProcess()
    except TimeoutError:
      print('Haven\'t heard from the server for too long, Exiting')
    except (KeyboardInterrupt, DoneError):
      print('Finished')
    finally:
      try:
        self._comm.sendMsg('(bye)')
      except:
        pass
      for p in self._agentPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      for p in self._npcPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      self._comm.close()
      self.printStats()
Exemplo n.º 2
0
class Trainer(object):
  """ Trainer is responsible for setting up the players and game.
  """
  def __init__(self, args, rng=numpy.random.RandomState(), server_port=6001,
               coach_port=6002):
    self._rng = rng # The Random Number Generator
    self._serverPort = server_port  # The port the server is listening on
    self._coachPort = coach_port # The coach port to talk with the server
    self._logDir = args.logDir # Directory to store logs
    self._record = args.record # Record states + actions
    self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
    self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
    self._maxTrials = args.numTrials # Maximum number of trials to play
    self._maxFrames = args.numFrames # Maximum number of frames to play
    self._maxFramesPerTrial = args.maxFramesPerTrial
    # =============== FIELD DIMENSIONS =============== #
    self.NUM_FRAMES_TO_HOLD = 2 # Hold ball this many frames to capture
    self.HOLD_FACTOR = 1.5 # Gain to calculate ball control
    self.PITCH_WIDTH = 68.0 # Width of the field
    self.PITCH_LENGTH = 105.0 # Length of field in long-direction
    self.UNTOUCHED_LENGTH = 100 # Trial will end if ball untouched for this long
    # allowedBallX, allowedBallY defines the usable area of the playfield
    self._allowedBallX = numpy.array([-0.1, 0.5 * self.PITCH_LENGTH])
    self._allowedBallY = numpy.array([-0.5 * self.PITCH_WIDTH, 0.5 * self.PITCH_WIDTH])
    # =============== COUNTERS =============== #
    self._numFrames = 0 # Number of frames seen in HFO trials
    self._frame = 0 # Current frame id
    self._lastTrialStart = -1 # Frame Id in which the last trial started
    self._lastFrameBallTouched = -1 # Frame Id in which ball was last touched
    # =============== TRIAL RESULTS =============== #
    self._numTrials = 0 # Total number of HFO trials
    self._numGoals = 0 # Trials in which the offense scored a goal
    self._numBallsCaptured = 0 # Trials in which defense captured the ball
    self._numBallsOOB = 0 # Trials in which ball went out of bounds
    self._numOutOfTime = 0 # Trials that ran out of time
    # =============== AGENT =============== #
    self._numAgents = args.offenseAgents + args.defenseAgents
    self._offenseAgents = args.offenseAgents
    self._defenseAgents = args.defenseAgents
    self._agentTeams = [] # Names of the teams the agents are playing for
    self._agentNumInt = [] # List of agents internal team numbers
    self._agentNumExt = [] # List of agents external team numbers
    self._agentServerPort = args.port # Base Port for agent's server
    self._agentOnBall = args.agent_on_ball # If true, agent starts with the ball
    # =============== MISC =============== #
    self._offenseTeamName = '' # Name of the offensive team
    self._defenseTeamName = '' # Name of the defensive team
    self._playerPositions = numpy.zeros((11,2,2)) # Positions of the players
    self._ballPosition = numpy.zeros(2) # Position of the ball
    self._ballHeld = numpy.zeros((11,2)) # Track player holding the ball
    self._teams = [] # Team indexes for offensive and defensive teams
    self._SP = {} # Sever Parameters. Recieved when connecting to the server.
    self._isPlaying = False # Is a game being played?
    self._teamHoldingBall = None # Team currently in control of the ball
    self._playerHoldingBall = None # Player current in control of ball
    self._agentPopen = [] # Agent's processes
    self.initMsgHandlers()

  def launch_agent(self, agent_num, play_offense, port):
    """Launch the learning agent using the start_agent.sh script and
    return a DummyPopen for the process.

    """
    print '[Trainer] Launching Agent', str(agent_num)
    if play_offense:
      assert self._numOffense > 0
      team_name = self._offenseTeamName
      self._agentTeams.append(team_name)
      # First offense number is reserved for inactive offensive goalie
      internal_player_num = agent_num + 1
      self._agentNumInt.append(internal_player_num)
      numTeammates = self._numOffense - 1
      numOpponents = self._numDefense
    else:
      assert self._numDefense > 0
      team_name = self._defenseTeamName
      self._agentTeams.append(team_name)
      internal_player_num = agent_num
      self._agentNumInt.append(internal_player_num)
      numTeammates = self._numDefense - 1
      numOpponents = self._numOffense
    ext_num = self.convertToExtPlayer(team_name, internal_player_num)
    self._agentNumExt.append(ext_num)
    binary_dir = os.path.dirname(os.path.realpath(__file__))
    agentCmd = 'start_agent.sh -t %s -u %i -p %i -P %i --log-dir %s'\
               ' --numTeammates %i --numOpponents %i'\
               ' --playingOffense %i --serverPort %i'\
               %(team_name, ext_num, self._serverPort,
                 self._coachPort, self._logDir, numTeammates,
                 numOpponents, play_offense, port)
    if self._record:
      agentCmd += ' --record'
    agentCmd = os.path.join(binary_dir, agentCmd)
    agentCmd = agentCmd.split(' ')
    # Ignore stderr because librcsc continually prints to it
    kwargs = {}#{'stderr':open('/dev/null','w')}
    p = subprocess.Popen(agentCmd, **kwargs)
    p.wait()
    pid_file = os.path.join(self._logDir, 'start%i'%p.pid)
    print '[Trainer] Parsing agent\'s pid from file:', pid_file
    assert os.path.isfile(pid_file)
    with open(pid_file,'r') as f:
      output = f.read()
    pid = int(re.findall('PID: (\d+)',output)[0])
    return DummyPopen(pid)

  def getDefensiveRoster(self, team_name):
    """Returns a list of player numbers on a given team that are thought
    to prefer defense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
    if team_name == 'Borregos':
      return [9,10,8,11,7,4,6,2,3,5]
    elif team_name == 'WrightEagle':
      return [5,2,8,9,10,6,3,11,4,7]
    else:
      return [2,3,4,5,6,7,8,11,9,10]

  def getOffensiveRoster(self, team_name):
    """Returns a list of player numbers on a given team that are thought
    to prefer offense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
    if team_name == 'Borregos':
      return [2,4,6,5,3,7,9,10,8,11]
    elif team_name == 'WrightEagle':
      return [11,4,7,3,6,10,8,9,2,5]
    else:
      return [11,7,8,9,10,6,3,2,4,5]

  def setTeams(self):
    """ Sets the offensive and defensive teams and player rosters. """
    self._offenseTeamInd = 0
    self._offenseTeamName = self._teams[self._offenseTeamInd]
    self._defenseTeamName = self._teams[1-self._offenseTeamInd]
    offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
    defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
    self._offenseOrder = [1] + offensive_roster # 1 for goalie
    self._defenseOrder = [1] + defensive_roster # 1 for goalie

  def teamToInd(self, team_name):
    """ Returns the index of a given team. """
    return self._teams.index(team_name)

  def parseMsg(self, msg):
    """ Parse a message """
    assert(msg[0] == '(')
    res, ind = self.__parseMsg(msg,1)
    assert(ind == len(msg)), msg
    return res

  def __parseMsg(self, msg, ind):
    """ Recursively parse a message. """
    res = []
    while True:
      if msg[ind] == '"':
        ind += 1
        startInd = ind
        while msg[ind] != '"':
          ind += 1
        res.append(msg[startInd:ind])
        ind += 1
      elif msg[ind] == '(':
        inner,ind = self.__parseMsg(msg,ind+1)
        res.append(inner)
      elif msg[ind] == ' ':
        ind += 1
      elif msg[ind] == ')':
        return res,ind+1
      else:
        startInd = ind
        while msg[ind] not in ' ()':
          ind += 1
        res.append(msg[startInd:ind])

  def initComm(self):
    """ Initialize communication to server. """
    self._comm = ClientCommunicator(port=self._coachPort)
    self.send('(init (version 8.0))')
    self.checkMsg('(init ok)', retryCount=5)
    # self.send('(eye on)')
    self.send('(ear on)')

  def _hear(self, body):
    """ Handle a hear message. """
    timestep,playerInfo,msg = body
    if len(playerInfo) != 3:
      return
    _,team,player = playerInfo
    try:
      length = int(msg[0])
    except:
      return
    msg = msg[1:length+1]
    if msg == 'START':
      if self._isPlaying:
        print '[Trainer] Already playing, ignoring message'
      else:
        self.startGame()
    elif msg == 'DONE':
      raise DoneError
    else:
      print '[Trainer] Unhandled message from agent: %s' % msg

  def initMsgHandlers(self):
    """ Create handlers for different messages. """
    self._msgHandlers = []
    self.ignoreMsg('player_param')
    self.ignoreMsg('player_type')
    self.ignoreMsg('ok','change_mode')
    self.ignoreMsg('ok','ear')
    self.ignoreMsg('ok','move')
    self.ignoreMsg('ok','recover')
    self.ignoreMsg('ok','say')
    self.registerMsgHandler(self._handleSP,'server_param')
    self.registerMsgHandler(self._hear,'hear')

  def recv(self, retryCount=None):
    """ Recieve a message. Retry a specified number of times. """
    return self._comm.recvMsg(retryCount=retryCount).strip()

  def send(self, msg):
    """ Send a message. """
    self._comm.sendMsg(msg)

  def checkMsg(self, expectedMsg, retryCount=None):
    """ Check that the next message is same as expected message. """
    msg = self.recv(retryCount)
    if msg != expectedMsg:
      print >>sys.stderr,'[Trainer] Error with message'
      print >>sys.stderr,'  expected: %s' % expectedMsg
      print >>sys.stderr,'  received: %s' % msg
      # print >>sys.stderr,len(expectedMsg),len(msg)
      raise ValueError

  def extractPoint(self, msg):
    """ Extract a point from the provided message. """
    return numpy.array(map(float,msg[:2]))

  def convertToExtPlayer(self, team, num):
    """ Returns the external player number for a given player. """
    assert team == self._offenseTeamName or team == self._defenseTeamName,\
      'Invalid team name %s. Valid choices: %s, %s.'\
      %(team, self._offenseTeamName, self._defenseTeamName)
    if team == self._offenseTeamName:
      return self._offenseOrder[num]
    else:
      return self._defenseOrder[num]

  def convertFromExtPlayer(self, team, num):
    """ Maps external player number to internal player number. """
    if team == self._offenseTeamName:
      return self._offenseOrder.index(num)
    else:
      return self._defenseOrder.index(num)

  def seeGlobal(self, body):
    """Send a look message to extract global information on ball and
    player positions.

    """
    self.send('(look)')
    self._frame = int(body[0])
    for obj in body[1:]:
      objType = obj[0]
      objData = obj[1:]
      if objType[0] == 'g':
        continue
      elif objType[0] == 'b':
        self._ballPosition = self.extractPoint(objData)
      elif objType[0] == 'p':
        teamName = objType[1]
        team = self.teamToInd(teamName)
        playerNum = self.convertFromExtPlayer(teamName,int(objType[2]))
        self._playerPositions[playerNum,:,team] = self.extractPoint(objData)

  def registerMsgHandler(self,handler,*args,**kwargs):
    '''Register a message handler.

    Handler will be called on a message that matches *args.

    '''
    args = list(args)
    i,_,_ = self._findHandlerInd(args)
    if i < 0:
      self._msgHandlers.append([args,handler])
    else:
      if ('quiet' not in kwargs) or (not kwargs['quiet']):
        print '[Trainer] Updating handler for %s' % (' '.join(args))
      self._msgHandlers[i] = [args,handler]

  def unregisterMsgHandler(self, *args):
    """ Delete a message handler. """
    i,_,_ = self._findHandlerInd(args)
    assert(i >= 0)
    del self._msgHandlers[i]

  def _findHandlerInd(self, msg):
    """ Find the handler for a particular message. """
    msg = list(msg)
    for i,(partial,handler) in enumerate(self._msgHandlers):
      recPartial = msg[:len(partial)]
      if recPartial == partial:
        return i,len(partial),handler
    return -1,None,None

  def handleMsg(self, msg):
    """ Handle a message using the registered handlers. """
    i,prefixLength,handler = self._findHandlerInd(msg)
    if i < 0:
      print '[Trainer] Unhandled message:',msg[0:2]
    else:
      handler(msg[prefixLength:])

  def ignoreMsg(self,*args,**kwargs):
    """ Ignore a certain type of message. """
    self.registerMsgHandler(lambda x: None,*args,**kwargs)

  def _handleSP(self, body):
    """ Handler for the sever params message. """
    for param in body:
      try:
        val = int(param[1])
      except:
        try:
          val = float(param[1])
        except:
          val = param[1]
      self._SP[param[0]] = val

  def listenAndProcess(self):
    """ Gather messages and process them. """
    msg = self.recv()
    assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
    msg = self.parseMsg(msg)
    self.handleMsg(msg)

  def _readTeamNames(self,body):
    """ Read the names of each of the teams. """
    self._teams = []
    for _,_,team in body:
      self._teams.append(team)
    time.sleep(0.1)
    self.send('(team_names)')

  def waitOnTeam(self, first):
    """Wait on a given team. First indicates if this is the first team
    connected or the second.

    """
    self.send('(team_names)')
    partial = ['ok','team_names']
    self.registerMsgHandler(self._readTeamNames,*partial,quiet=True)
    while len(self._teams) < (1 if first else 2):
      self.listenAndProcess()
    #self.unregisterMsgHandler(*partial)
    self.ignoreMsg(*partial,quiet=True)

  def checkIfAllPlayersConnected(self):
    """ Returns true if all players are connected. """
    self.send('(look)')
    partial = ['ok','look']
    self._numPlayers = 0
    def f(x):
      self._numPlayers = len(x) - 4 # -4 for time, ball, goal_l, and goal_r
      self.send('(look)')
    self.registerMsgHandler(f,*partial)
    while self._numPlayers != 2 * 11:
      self.listenAndProcess()
    self.ignoreMsg(*partial,quiet=True)

  def startGame(self):
    """ Starts a game of HFO. """
    self.reset()
    self.registerMsgHandler(self.seeGlobal, 'see_global')
    self.registerMsgHandler(self.seeGlobal, 'ok', 'look', quiet=True)
    #self.registerMsgHandler(self.checkBall,'ok','check_ball')
    self.send('(look)')
    self._isPlaying = True

  def calcBallHolder(self):
    '''Calculates the ball holder, returns results in teamInd, playerInd. '''
    totalHeld = 0
    for team in self._teams:
      for i in range(11):
        pos = self._playerPositions[i,:,self.teamToInd(team)]
        distBound = self._SP['kickable_margin'] + self._SP['player_size'] \
                    + self._SP['ball_size']
        distBound *= self.HOLD_FACTOR
        if numpy.linalg.norm(self._ballPosition - pos) < distBound:
          self._ballHeld[i,self.teamToInd(team)] += 1
          totalHeld += 1
        else:
          self._ballHeld[i,self.teamToInd(team)] = 0
    # If multiple players are close to the ball, no-one is holding
    if totalHeld > 1:
      self._ballHeld[:,:] = 0
    inds = numpy.transpose((self._ballHeld >= self.NUM_FRAMES_TO_HOLD).nonzero())
    assert(len(inds) <= 1)
    if len(inds) == 1:
      return inds[0,1],inds[0,0]
    else:
      return None,None

  def isGoal(self):
    """ Returns true if a goal has been scored. """
    return (self._ballPosition[0] > self._allowedBallX[1]) \
      and (numpy.abs(self._ballPosition[1]) <= 0.5 * self._SP['goal_width'])

  def isOOB(self):
    """ Returns true if the ball is out of bounds. """
    return self._ballPosition[0] < self._allowedBallX[0] \
      or self._ballPosition[0] > self._allowedBallX[1] \
      or self._ballPosition[1] < self._allowedBallY[0] \
      or self._ballPosition[1] > self._allowedBallY[1]

  def isCaptured(self):
    """ Returns true if the ball is captured by defense. """
    return self._teamHoldingBall not in [None,self._offenseTeamInd]

  def isOOT(self):
    """ Returns true if the trial has run out of time. """
    return self._frame - self._lastFrameBallTouched > self.UNTOUCHED_LENGTH \
      or (self._maxFramesPerTrial > 0 and self._frame -
          self._lastTrialStart > self._maxFramesPerTrial)

  def movePlayer(self, team, internal_num, pos, convertToExt=True):
    """ Move a player to a specified position.
    Args:
      team: the team name of the player
      interal_num: the player's internal number
      pos: position to move player to
      convertToExt: convert interal player num to external
    """
    num = self.convertToExtPlayer(team, internal_num) if convertToExt \
          else internal_num
    self.send('(move (player %s %i) %f %f)' % (team, num, pos[0], pos[1]))

  def moveBall(self, pos):
    """ Moves the ball to a specified x,y position. """
    self.send('(move (ball) %f %f 0.0 0.0 0.0)' % tuple(pos))

  def randomPointInBounds(self, xBounds=None, yBounds=None):
    """Returns a random point inside of the box defined by xBounds,
    yBounds. Where xBounds=[x_min, x_max] and yBounds=[y_min,
    y_max]. Defaults to the xy-bounds of the playable HFO area.

    """
    if xBounds is None:
      xBounds = self.allowedBallX
    if yBounds is None:
      yBounds = self.allowedBallY
    pos = numpy.zeros(2)
    bounds = [xBounds, yBounds]
    for i in range(2):
      pos[i] = self._rng.rand() * (bounds[i][1] - bounds[i][0]) + bounds[i][0]
    return pos

  def boundPoint(self, pos):
    """Ensures a point is within the minimum and maximum bounds of the
    HFO playing area.

    """
    pos[0] = min(max(pos[0], self._allowedBallX[0]), self._allowedBallX[1])
    pos[1] = min(max(pos[1], self._allowedBallY[0]), self._allowedBallY[1])
    return pos

  def reset(self):
    """ Resets the HFO domain by moving the ball and players. """
    self.resetBallPosition()
    self.resetPlayerPositions()
    self.send('(recover)')
    self.send('(change_mode play_on)')
    # self.send('(say RESET)')

  def resetBallPosition(self):
    """Reset the position of the ball for a new HFO trial. """
    self._ballPosition = self.boundPoint(self.randomPointInBounds(
      .2*self._allowedBallX+.05*self.PITCH_LENGTH, .8*self._allowedBallY))
    self.moveBall(self._ballPosition)

  def getOffensiveResetPosition(self):
    """ Returns a random position for an offensive player. """
    offsets = [
      [-1,-1],
      [-1,1],
      [1,1],
      [1,-1],
      [0,2],
      [0,-2],
      [-2,-2],
      [-2,2],
      [2,2],
      [2,-2],
    ]
    offset = offsets[self._rng.randint(len(offsets))]
    offset_from_ball = 0.1 * self.PITCH_LENGTH * self._rng.rand(2) + \
                       0.1 * self.PITCH_LENGTH * numpy.array(offset)
    return self.boundPoint(self._ballPosition + offset_from_ball)
    # return self._ballPosition

  def getDefensiveResetPosition(self):
    """ Returns a random position for a defensive player. """
    return self.boundPoint(self.randomPointInBounds(
      [0.5 * 0.5 * self.PITCH_LENGTH, 0.75 * 0.5 * self.PITCH_LENGTH],
      0.8 * self._allowedBallY))

  def resetPlayerPositions(self):
    """Reset the positions of the players. This is called after a trial
    ends to setup for the next trial.

    """
    # Always Move the offensive goalie to the left goal
    self.movePlayer(self._offenseTeamName, 0, [-0.5 * self.PITCH_LENGTH, 0])
    # Move the rest of the offense
    for i in xrange(1, self._numOffense + 1):
      self.movePlayer(self._offenseTeamName, i, self.getOffensiveResetPosition())
    # Move the agent to the ball
    if self._agentOnBall and self._offenseAgents > 0:
      self.movePlayer(self._offenseTeamName, self._agentNumInt[0], self._ballPosition)
    # Move the defensive goalie
    if self._numDefense > 0:
      self.movePlayer(self._defenseTeamName, 0, [0.5 * self.PITCH_LENGTH,0])
    # Move the rest of the defense
    for i in xrange(1, self._numDefense):
      self.movePlayer(self._defenseTeamName, i, self.getDefensiveResetPosition())

  def removeNonHFOPlayers(self):
    """Removes players that aren't involved in HFO game.

    The players whose numbers are greater than numOffense/numDefense
    are sent to left-field.

    """
    offensive_agent_numbers = self._agentNumInt[:self._offenseAgents]
    defensive_agent_numbers = self._agentNumInt[self._offenseAgents:]
    for i in xrange(self._numOffense + 1, 11):
      if i not in offensive_agent_numbers:
        self.movePlayer(self._offenseTeamName, i, [-0.25 * self.PITCH_LENGTH, 0])
    for i in xrange(self._numDefense, 11):
      if i not in defensive_agent_numbers:
        self.movePlayer(self._defenseTeamName, i, [-0.25 * self.PITCH_LENGTH, 0])

  def step(self):
    """ Takes a simulated step. """
    # self.send('(check_ball)')
    self.removeNonHFOPlayers()
    self._teamHoldingBall, self._playerHoldingBall = self.calcBallHolder()
    if self._teamHoldingBall is not None:
      self._lastFrameBallTouched = self._frame
    if self.trialOver():
      self.updateResults()
      self._lastFrameBallTouched = self._frame
      self.reset()

  def updateResults(self):
    """ Updates the various members after a trial has ended. """
    if self.isGoal():
      self._numGoals += 1
      result = 'Goal'
      self.send('(say GOAL)')
    elif self.isOOB():
      self._numBallsOOB += 1
      result = 'Out of Bounds'
      self.send('(say OUT_OF_BOUNDS)')
    elif self.isCaptured():
      self._numBallsCaptured += 1
      result = 'Defense Captured'
      self.send('(say CAPTURED_BY_DEFENSE)')
    elif self.isOOT():
      self._numOutOfTime += 1
      result = 'Ball untouched for too long'
      self.send('(say OUT_OF_TIME)')
    else:
      print '[Trainer] Error: Unable to detect reason for End of Trial!'
      sys.exit(1)
    print '[Trainer] EndOfTrial: %2i /%2i %5i %s'%\
      (self._numGoals, self._numTrials, self._frame, result)
    self._numTrials += 1
    self._numFrames += self._frame - self._lastTrialStart
    self._lastTrialStart = self._frame
    if (self._maxTrials > 0) and (self._numTrials >= self._maxTrials):
      raise DoneError
    if (self._maxFrames > 0) and (self._numFrames >= self._maxFrames):
      raise DoneError

  def trialOver(self):
    """Returns true if the trial has ended for one of the following
    reasons: Goal scored, out of bounds, captured by defense, or
    untouched for too long.

    """
    # The trial is still being setup, it cannot be over.
    if self._frame - self._lastTrialStart < 5:
      return False
    return self.isGoal() or self.isOOB() or self.isCaptured() or self.isOOT()

  def printStats(self):
    print 'Num frames in completed trials : %i' % (self._numFrames)
    print 'Trials             : %i' % self._numTrials
    print 'Goals              : %i' % self._numGoals
    print 'Defense Captured   : %i' % self._numBallsCaptured
    print 'Balls Out of Bounds: %i' % self._numBallsOOB
    print 'Out of Time        : %i' % self._numOutOfTime

  def checkLive(self, necProcesses):
    """Returns true if each of the necessary processes is still alive and
    running.

    """
    for p,name in necProcesses:
      if p.poll() is not None:
        print '[Trainer] Something necessary closed (%s), exiting' % name
        return False
    return True

  def run(self, necProcesses):
    """ Run the trainer """
    try:
      for agent_num in xrange(self._offenseAgents):
        port = self._agentServerPort + agent_num
        agent = self.launch_agent(agent_num, play_offense=True, port=port)
        self._agentPopen.append(agent)
        necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
      for agent_num in xrange(self._defenseAgents):
        port = self._agentServerPort + agent_num + self._offenseAgents
        agent = self.launch_agent(agent_num, play_offense=False, port=port)
        self._agentPopen.append(agent)
        necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
      # Broadcast the HFO configuration
      offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
                               for i in xrange(1, self._numOffense + 1)])
      defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
                               for i in xrange(self._numDefense)])
      self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
                  ' num_defense %d offense_nums %s defense_nums %s)'
                %(self._offenseTeamName, self._defenseTeamName,
                  self._numOffense, self._numDefense,
                  offense_nums, defense_nums))
      self.startGame()
      while self.checkLive(necProcesses):
        prevFrame = self._frame
        self.listenAndProcess()
        if self._frame != prevFrame:
          self.step()
    except TimeoutError:
      print '[Trainer] Haven\'t heard from the server for too long, Exiting'
    except (KeyboardInterrupt, DoneError):
      print '[Trainer] Exiting'
    finally:
      for p in self._agentPopen:
        p.send_signal(SIGINT)
      try:
        self._comm.sendMsg('(bye)')
      except:
        pass
      self._comm.close()
      self.printStats()
Exemplo n.º 3
0
Arquivo: Trainer.py Projeto: cowhi/HFO
class Trainer(object):
  """ Trainer is responsible for setting up the players and game.
  """
  def __init__(self, args, server_port=6001, coach_port=6002):
    self._serverPort = server_port  # The port the server is listening on
    self._coachPort = coach_port # The coach port to talk with the server
    self._logDir = args.logDir # Directory to store logs
    self._record = args.record # Record states + actions
    self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
    self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
    # =============== COUNTERS =============== #
    self._numFrames = 0 # Number of frames seen in HFO trials
    self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
    self._frame = 0 # Current frame id
    self._lastTrialStart = -1 # Frame Id in which the last trial started
    # =============== TRIAL RESULTS =============== #
    self._numTrials = 0 # Total number of HFO trials
    self._numGoals = 0 # Trials in which the offense scored a goal
    self._numBallsCaptured = 0 # Trials in which defense captured the ball
    self._numBallsOOB = 0 # Trials in which ball went out of bounds
    self._numOutOfTime = 0 # Trials that ran out of time
    # =============== AGENT =============== #
    self._numAgents = args.offenseAgents + args.defenseAgents
    self._offenseAgents = args.offenseAgents
    self._defenseAgents = args.defenseAgents
    self._agentReady = set([]) # Unums of ready agents
    self._agentTeams = [] # Names of the teams the agents are playing for
    self._agentNumInt = [] # List of agents internal team numbers
    self._agentServerPort = args.port # Base Port for agent's server
    # =============== MISC =============== #
    self._offenseTeamName = '' # Name of the offensive team
    self._defenseTeamName = '' # Name of the defensive team
    self._isPlaying = False # Is a game being played?
    self._done = False # Are we finished?
    self._agentPopen = [] # Agent's processes
    self._npcPopen = [] # NPC's processes
    self._connectedPlayers = [] # List of connected players
    self.initMsgHandlers()

  def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
    """Launches a learning agent using the agent binary

    Returns a Popen process object
    """
    if play_offense:
      assert self._numOffense > 0
      team_name = self._offenseTeamName
      self._agentTeams.append(team_name)
      # First offense number is reserved for inactive offensive goalie
      internal_player_num = agent_num + 1
      self._agentNumInt.append(internal_player_num)
    else:
      assert self._numDefense > 0
      team_name = self._defenseTeamName
      self._agentTeams.append(team_name)
      internal_player_num = agent_num
      self._agentNumInt.append(internal_player_num)
    binary_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                              'teams', 'base')
    config_dir = os.path.join(binary_dir, 'config/formations-dt')
    print("Waiting for player-controlled agent %s-%d: config_dir=%s, "\
          "server_port=%d, server_addr=%s, team_name=%s, play_goalie=%r"
          % (self._offenseTeamName if play_offense else self._defenseTeamName,
             agent_num, config_dir, self._serverPort, "localhost", team_name,
             agent_ext_num==1))
    if wait_until_join:
      self.waitOnPlayer(agent_ext_num, play_offense)
    return None

  def createTeam(self, requested_team_name, play_offense):
    """ Given a team name, returns the team object. """
    teams_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'teams')
    if requested_team_name == 'helios':
      print 'Creating team Helios'
      team_name = 'HELIOS_' + ('left' if play_offense else 'right')
      team_dir = os.path.join(teams_dir, 'helios', 'helios-13Eindhoven')
      lib_dir = os.path.join(teams_dir, 'helios', 'local', 'lib')
      return Teams.Helios(team_name, team_dir, lib_dir,
                          binaryName='helios_player', host='localhost',
                          port=self._serverPort)
    elif requested_team_name == 'base':
      print 'Creating team Agent2d (base)'
      team_name = 'base_' + ('left' if play_offense else 'right')
      team_dir = os.path.join(teams_dir, 'base')
      lib_dir = None
      return Teams.Agent2d(team_name, team_dir, lib_dir,
                           binaryName='sample_player', logDir=self._logDir,
                           record=self._record, host='localhost',
                           port=self._serverPort)
    else:
      print 'Unknown team requested: ' + requested_team_name
      sys.exit(1)

  def getTeams(self, offense_team_name, defense_team_name):
    """ Sets the offensive and defensive teams and player rosters. """
    # Set up offense team
    offenseTeam = self.createTeam(offense_team_name, play_offense=True)
    self._offenseTeamName = offenseTeam._name
    self._offenseOrder = [1] + offenseTeam._offense_order # 1 for goalie
    # Set up defense team
    defenseTeam = self.createTeam(defense_team_name, play_offense=False)
    self._defenseTeamName = defenseTeam._name
    self._defenseOrder = [1] + defenseTeam._defense_order # 1 for goalie
    return (offenseTeam, defenseTeam)

  def parseMsg(self, msg):
    """ Parse a message """
    assert(msg[0] == '(')
    res, ind = self.__parseMsg(msg,1)
    assert(ind == len(msg)), msg
    return res

  def __parseMsg(self, msg, ind):
    """ Recursively parse a message. """
    res = []
    while True:
      if msg[ind] == '"':
        ind += 1
        startInd = ind
        while msg[ind] != '"':
          ind += 1
        res.append(msg[startInd:ind])
        ind += 1
      elif msg[ind] == '(':
        inner,ind = self.__parseMsg(msg,ind+1)
        res.append(inner)
      elif msg[ind] == ' ':
        ind += 1
      elif msg[ind] == ')':
        return res,ind+1
      else:
        startInd = ind
        while msg[ind] not in ' ()':
          ind += 1
        res.append(msg[startInd:ind])

  def initComm(self):
    """ Initialize communication to server. """
    self._comm = ClientCommunicator(port=self._coachPort)
    self.send('(init (version 8.0))')
    self.checkMsg('(init ok)', retryCount=5)
    # self.send('(eye on)')
    self.send('(ear on)')

  def _hearRef(self, body):
    """ Handles hear messages from referee. """
    assert body[0] == 'referee', 'Expected referee message.'
    _,ts,event = body
    self._frame = int(ts)
    endOfTrial = False
    if 'GOAL' in event:
      self._numGoals += 1
      self._numGoalFrames += self._frame - self._lastTrialStart
      endOfTrial = True
    elif event == 'OUT_OF_BOUNDS':
      self._numBallsOOB += 1
      endOfTrial = True
    elif 'CAPTURED_BY_DEFENSE' in event:
      self._numBallsCaptured += 1
      endOfTrial = True
    elif event == 'OUT_OF_TIME':
      self._numOutOfTime += 1
      endOfTrial = True
    elif event == 'HFO_FINISHED':
      self._done = True
    if endOfTrial:
      self._numTrials += 1
      print 'EndOfTrial: %d / %d %d %s'%\
        (self._numGoals, self._numTrials, self._frame, event)
      self._numFrames += self._frame - self._lastTrialStart
      self._lastTrialStart = self._frame
      self.getConnectedPlayers()

  def _hear(self, body):
    """ Handle a hear message. """
    if body[0] == 'referee':
      self._hearRef(body)
      return
    timestep,playerInfo,msg = body
    try:
      _,team,player = playerInfo[:3]
      length = int(msg[0])
    except:
      return
    msg = msg[1:length+1]
    if msg == 'START':
      if self._isPlaying:
        print 'Already playing, ignoring message'
      else:
        self.startGame()
    elif msg == 'DONE':
      raise DoneError
    elif msg == 'ready':
      print 'Agent Connected:', team, player
      self._agentReady.add((team, player))
    else:
      print 'Unhandled message from agent: %s' % msg

  def initMsgHandlers(self):
    """ Create handlers for different messages. """
    self._msgHandlers = []
    self.ignoreMsg('player_param')
    self.ignoreMsg('player_type')
    self.ignoreMsg('ok','change_mode')
    self.ignoreMsg('ok','ear')
    self.ignoreMsg('ok','move')
    self.ignoreMsg('ok','recover')
    self.ignoreMsg('ok','say')
    self.ignoreMsg('server_param')
    self.registerMsgHandler(self._hear,'hear')

  def recv(self, retryCount=None):
    """ Recieve a message. Retry a specified number of times. """
    return self._comm.recvMsg(retryCount=retryCount).strip()

  def send(self, msg):
    """ Send a message. """
    self._comm.sendMsg(msg)

  def checkMsg(self, expectedMsg, retryCount=None):
    """ Check that the next message is same as expected message. """
    msg = self.recv(retryCount)
    if msg != expectedMsg:
      print >>sys.stderr,'Error with message'
      print >>sys.stderr,'  expected: %s' % expectedMsg
      print >>sys.stderr,'  received: %s' % msg
      # print >>sys.stderr,len(expectedMsg),len(msg)
      raise ValueError

  def convertToExtPlayer(self, team, num):
    """ Returns the external player number for a given player. """
    assert team == self._offenseTeamName or team == self._defenseTeamName,\
      'Invalid team name %s. Valid choices: %s, %s.'\
      %(team, self._offenseTeamName, self._defenseTeamName)
    if team == self._offenseTeamName:
      return self._offenseOrder[num]
    else:
      return self._defenseOrder[num]

  def registerMsgHandler(self,handler,*args,**kwargs):
    '''Register a message handler.

    Handler will be called on a message that matches *args.

    '''
    args = list(args)
    i,_,_ = self._findHandlerInd(args)
    if i < 0:
      self._msgHandlers.append([args,handler])
    else:
      if ('quiet' not in kwargs) or (not kwargs['quiet']):
        print 'Updating handler for %s' % (' '.join(args))
      self._msgHandlers[i] = [args,handler]

  def unregisterMsgHandler(self, *args):
    """ Delete a message handler. """
    i,_,_ = self._findHandlerInd(args)
    assert(i >= 0)
    del self._msgHandlers[i]

  def _findHandlerInd(self, msg):
    """ Find the handler for a particular message. """
    msg = list(msg)
    for i,(partial,handler) in enumerate(self._msgHandlers):
      recPartial = msg[:len(partial)]
      if recPartial == partial:
        return i,len(partial),handler
    return -1,None,None

  def handleMsg(self, msg):
    """ Handle a message using the registered handlers. """
    i,prefixLength,handler = self._findHandlerInd(msg)
    if i < 0:
      print 'Unhandled message:',msg[0:2]
    else:
      handler(msg[prefixLength:])

  def ignoreMsg(self,*args,**kwargs):
    """ Ignore a certain type of message. """
    self.registerMsgHandler(lambda x: None,*args,**kwargs)

  def listenAndProcess(self, retry_count=None):
    """ Gather messages and process them. """
    try:
      msg = self.recv(retry_count)
      assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
      msg = self.parseMsg(msg)
      self.handleMsg(msg)
    except TimeoutError:
      pass

  def disconnectPlayer(self, player, player_num, on_offense):
    """Wait on a launched player to disconnect from the server. """
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    while (team_name, str(player_num)) in self._connectedPlayers:
      self.send('(disconnect_player %s %d)'%(team_name, player_num))
      self.getConnectedPlayers()
    player.kill()

  def getConnectedPlayers(self):
    """ Get the list of connected players. Populates self._connectedPlayers. """
    self._gotLook = False
    self.send('(look)')
    partial = ['ok','look']
    def f(body):
      self._gotLook = True
      del self._connectedPlayers[:]
      for i in xrange(4, len(body)):
        _,team,num = body[i][0][:3]
        if (team, num) not in self._connectedPlayers:
          self._connectedPlayers.append((team,num))
    self.registerMsgHandler(f,*partial,quiet=True)
    while not self._gotLook:
      self.listenAndProcess()
      self.send('(look)')
    self.ignoreMsg(*partial,quiet=True)

  def waitOnPlayer(self, player_num, on_offense):
    """ Wait on a launched player to connect and be reported by the server. """
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    while (team_name, str(player_num)) not in self._connectedPlayers:
      self.getConnectedPlayers()

  def allPlayersConnected(self):
    """ Returns true all players are connected. """
    return len(self._connectedPlayers) == self._numOffense + self._numDefense

  def sendHFOConfig(self):
    """ Broadcast the HFO configuration """
    offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
                             for i in xrange(1, self._numOffense + 1)])
    defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
                             for i in xrange(self._numDefense)])
    self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
                ' num_defense %d offense_nums %s defense_nums %s)'
              %(self._offenseTeamName, self._defenseTeamName,
                self._numOffense, self._numDefense,
                offense_nums, defense_nums))

  def startGame(self):
    """ Starts a game of HFO. """
    self.send('(change_mode play_on)')
    self._isPlaying = True

  def printStats(self):
    print 'TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
      %(self._numFrames,
        self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
        self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan'))
    print 'Trials             : %i' % self._numTrials
    print 'Goals              : %i' % self._numGoals
    print 'Defense Captured   : %i' % self._numBallsCaptured
    print 'Balls Out of Bounds: %i' % self._numBallsOOB
    print 'Out of Time        : %i' % self._numOutOfTime

  def checkLive(self, necProcesses):
    """Returns true if each of the necessary processes is still alive and
    running.

    """
    for p,name in necProcesses:
      if p is not None and p.poll() is not None:
        print 'Something necessary closed (%s), exiting' % name
        return False
    return True

  def run(self, necProcesses, offense_team_name, defense_team_name):
    """ Run the trainer """
    try:
      (offenseTeam, defenseTeam) = self.getTeams(offense_team_name, defense_team_name)
      offense_unums = self._offenseOrder[1: self._numOffense + 1]
      sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
      defense_unums = self._defenseOrder[: self._numDefense]
      sorted_defense_agent_unums = sorted(self._defenseOrder[:self._defenseAgents])

      # Launch offense
      agent_num = 0
      for player_num in xrange(1, 12):
        if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num
          agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
                                    play_offense=True, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = offenseTeam.launch_npc(player_num)
          self.waitOnPlayer(player_num, True)
          if player_num in offense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'offense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=True)

      # Launch defense
      agent_num = 0
      for player_num in xrange(1, 12):
        if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num + self._offenseAgents
          agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
                                    play_offense=False, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = defenseTeam.launch_npc(player_num)
          self.waitOnPlayer(player_num, False)
          if player_num in defense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'defense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=False)

      print 'Checking all players connected'
      while not self.allPlayersConnected():
        self.getConnectedPlayers()

      time.sleep(0.1)
      self.sendHFOConfig()

      print 'Starting game'
      time.sleep(0.1)
      self.startGame()
      while self.allPlayersConnected() and self.checkLive(necProcesses) and not self._done:
        prevFrame = self._frame
        self.listenAndProcess()
    except TimeoutError:
      print 'Haven\'t heard from the server for too long, Exiting'
    except (KeyboardInterrupt, DoneError):
      print 'Finished'
    finally:
      try:
        self._comm.sendMsg('(bye)')
      except:
        pass
      for p in self._agentPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      for p in self._npcPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      self._comm.close()
      self.printStats()
Exemplo n.º 4
0
class Trainer(object):
  """ Trainer is responsible for setting up the players and game.
  """
  def __init__(self, args, server_port=6001, coach_port=6002):
    self._serverPort = server_port  # The port the server is listening on
    self._coachPort = coach_port # The coach port to talk with the server
    self._logDir = args.logDir # Directory to store logs
    self._record = args.record # Record states + actions
    self._numOffense = args.offenseAgents + args.offenseNPCs # Number offensive players
    self._numDefense = args.defenseAgents + args.defenseNPCs # Number defensive players
    # =============== COUNTERS =============== #
    self._numFrames = 0 # Number of frames seen in HFO trials
    self._numGoalFrames = 0 # Number of frames in goal-scoring HFO trials
    self._frame = 0 # Current frame id
    self._lastTrialStart = -1 # Frame Id in which the last trial started
    # =============== TRIAL RESULTS =============== #
    self._numTrials = 0 # Total number of HFO trials
    self._numGoals = 0 # Trials in which the offense scored a goal
    self._numBallsCaptured = 0 # Trials in which defense captured the ball
    self._numBallsOOB = 0 # Trials in which ball went out of bounds
    self._numOutOfTime = 0 # Trials that ran out of time
    # =============== AGENT =============== #
    self._numAgents = args.offenseAgents + args.defenseAgents
    self._offenseAgents = args.offenseAgents
    self._defenseAgents = args.defenseAgents
    self._agentReady = set([]) # Unums of ready agents
    self._agentTeams = [] # Names of the teams the agents are playing for
    self._agentNumInt = [] # List of agents internal team numbers
    self._agentServerPort = args.port # Base Port for agent's server
    # =============== MISC =============== #
    self._offenseTeamName = '' # Name of the offensive team
    self._defenseTeamName = '' # Name of the defensive team
    self._teams = [] # Team indexes for offensive and defensive teams
    self._isPlaying = False # Is a game being played?
    self._done = False # Are we finished?
    self._agentPopen = [] # Agent's processes
    self._npcPopen = [] # NPC's processes
    self._connectedPlayers = []
    self.initMsgHandlers()

  def launch_npc(self, player_num, play_offense, wait_until_join=True):
    """Launches a player using sample_player binary

    Returns a Popen process object
    """
    print 'Launch npc %s-%d'%(self._offenseTeamName if play_offense
                              else self._defenseTeamName, player_num)
    if play_offense:
      team_name = self._offenseTeamName
    else:
      team_name = self._defenseTeamName
    binary_dir = os.path.dirname(os.path.realpath(__file__))
    config_dir = os.path.join(binary_dir, '../config/formations-dt')
    player_conf = os.path.join(binary_dir, '../config/player.conf')
    player_cmd = os.path.join(binary_dir, 'sample_player')
    player_cmd += ' -t %s -p %i --config_dir %s ' \
                  ' --log_dir %s --player-config %s' \
                  %(team_name, self._serverPort, config_dir,
                    self._logDir, player_conf)
    if self._record:
        player_cmd += ' --record'
    if player_num == 1:
        player_cmd += ' -g'
    kwargs = {'stdout':open('/dev/null', 'w'),
              'stderr':open('/dev/null', 'w')}
    p = subprocess.Popen(player_cmd.split(' '), shell = False, **kwargs)
    if wait_until_join:
      self.waitOnPlayer(player_num, play_offense)
    return p

  def launch_agent(self, agent_num, agent_ext_num, play_offense, port, wait_until_join=True):
    """Launches a learning agent using the agent binary

    Returns a Popen process object
    """
    print 'Launch agent %s-%d'%(self._offenseTeamName if play_offense
                                else self._defenseTeamName, agent_num)
    if play_offense:
      assert self._numOffense > 0
      team_name = self._offenseTeamName
      self._agentTeams.append(team_name)
      # First offense number is reserved for inactive offensive goalie
      internal_player_num = agent_num + 1
      self._agentNumInt.append(internal_player_num)
      numTeammates = self._numOffense - 1
      numOpponents = self._numDefense
    else:
      assert self._numDefense > 0
      team_name = self._defenseTeamName
      self._agentTeams.append(team_name)
      internal_player_num = agent_num
      self._agentNumInt.append(internal_player_num)
      numTeammates = self._numDefense - 1
      numOpponents = self._numOffense
    binary_dir = os.path.dirname(os.path.realpath(__file__))
    config_dir = os.path.join(binary_dir, '../config/formations-dt')
    player_conf = os.path.join(binary_dir, '../config/player.conf')
    agent_cmd =  os.path.join(binary_dir, 'agent')
    agent_cmd += ' -t %s -p %i --numTeammates %i --numOpponents %i' \
                 ' --playingOffense %i --serverPort %i --log_dir %s' \
                 ' --player-config %s --config_dir %s' \
                 %(team_name, self._serverPort, numTeammates,
                   numOpponents, play_offense, port, self._logDir,
                   player_conf, config_dir)
    if agent_ext_num == 1:
      agent_cmd += ' -g'
    if self._record:
      agent_cmd += ' --record'
    # Comment next two lines to show output from agent.cpp and the server
    kwargs = {'stdout':open('/dev/null', 'w'),
              'stderr':open('/dev/null', 'w')}
    p = subprocess.Popen(agent_cmd.split(' '), shell = False, **kwargs)
    if wait_until_join:
      self.waitOnPlayer(agent_ext_num, play_offense)
    return p

  def getDefensiveRoster(self, team_name):
    """Returns a list of player numbers on a given team that are thought
    to prefer defense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
    if team_name == 'Borregos':
      return [9,10,8,11,7,4,6,2,3,5]
    elif team_name == 'WrightEagle':
      return [5,2,8,9,10,6,3,11,4,7]
    else:
      return [2,3,4,5,6,7,8,11,9,10]

  def getOffensiveRoster(self, team_name):
    """Returns a list of player numbers on a given team that are thought
    to prefer offense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
    if team_name == 'Borregos':
      return [2,4,6,5,3,7,9,10,8,11]
    elif team_name == 'WrightEagle':
      return [11,4,7,3,6,10,8,9,2,5]
    else:
      return [11,7,8,9,10,6,3,2,4,5]

  def addTeam(self, team_name):
    """ Adds a team to the team list"""
    self._teams.append(team_name)

  def setTeams(self):
    """ Sets the offensive and defensive teams and player rosters. """
    self._offenseTeamInd = 0
    self._offenseTeamName = self._teams[self._offenseTeamInd]
    self._defenseTeamName = self._teams[1-self._offenseTeamInd]
    offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
    defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
    self._offenseOrder = [1] + offensive_roster # 1 for goalie
    self._defenseOrder = [1] + defensive_roster # 1 for goalie

  def teamToInd(self, team_name):
    """ Returns the index of a given team. """
    return self._teams.index(team_name)

  def parseMsg(self, msg):
    """ Parse a message """
    assert(msg[0] == '(')
    res, ind = self.__parseMsg(msg,1)
    assert(ind == len(msg)), msg
    return res

  def __parseMsg(self, msg, ind):
    """ Recursively parse a message. """
    res = []
    while True:
      if msg[ind] == '"':
        ind += 1
        startInd = ind
        while msg[ind] != '"':
          ind += 1
        res.append(msg[startInd:ind])
        ind += 1
      elif msg[ind] == '(':
        inner,ind = self.__parseMsg(msg,ind+1)
        res.append(inner)
      elif msg[ind] == ' ':
        ind += 1
      elif msg[ind] == ')':
        return res,ind+1
      else:
        startInd = ind
        while msg[ind] not in ' ()':
          ind += 1
        res.append(msg[startInd:ind])

  def initComm(self):
    """ Initialize communication to server. """
    self._comm = ClientCommunicator(port=self._coachPort)
    self.send('(init (version 8.0))')
    self.checkMsg('(init ok)', retryCount=5)
    # self.send('(eye on)')
    self.send('(ear on)')

  def _hearRef(self, body):
    """ Handles hear messages from referee. """
    assert body[0] == 'referee', 'Expected referee message.'
    _,ts,event = body
    self._frame = int(ts)
    endOfTrial = False
    if 'GOAL' in event:
      self._numGoals += 1
      self._numGoalFrames += self._frame - self._lastTrialStart
      endOfTrial = True
    elif event == 'OUT_OF_BOUNDS':
      self._numBallsOOB += 1
      endOfTrial = True
    elif 'CAPTURED_BY_DEFENSE' in event:
      self._numBallsCaptured += 1
      endOfTrial = True
    elif event == 'OUT_OF_TIME':
      self._numOutOfTime += 1
      endOfTrial = True
    elif event == 'HFO_FINISHED':
      self._done = True
    if endOfTrial: 
      self._numTrials += 1
      print 'EndOfTrial: %d / %d %d %s'%\
        (self._numGoals, self._numTrials, self._frame, event)
      self._numFrames += self._frame - self._lastTrialStart
      self._lastTrialStart = self._frame

  def _hear(self, body):
    """ Handle a hear message. """
    if body[0] == 'referee':
      self._hearRef(body)
      return
    timestep,playerInfo,msg = body
    try:
      _,team,player = playerInfo[:3]
      length = int(msg[0])
    except:
      return
    msg = msg[1:length+1]
    if msg == 'START':
      if self._isPlaying:
        print 'Already playing, ignoring message'
      else:
        self.startGame()
    elif msg == 'DONE':
      raise DoneError
    elif msg == 'ready':
      print 'Agent Connected:', team, player
      self._agentReady.add((team, player))
    else:
      print 'Unhandled message from agent: %s' % msg

  def initMsgHandlers(self):
    """ Create handlers for different messages. """
    self._msgHandlers = []
    self.ignoreMsg('player_param')
    self.ignoreMsg('player_type')
    self.ignoreMsg('ok','change_mode')
    self.ignoreMsg('ok','ear')
    self.ignoreMsg('ok','move')
    self.ignoreMsg('ok','recover')
    self.ignoreMsg('ok','say')
    self.ignoreMsg('server_param')
    self.registerMsgHandler(self._hear,'hear')

  def recv(self, retryCount=None):
    """ Recieve a message. Retry a specified number of times. """
    return self._comm.recvMsg(retryCount=retryCount).strip()

  def send(self, msg):
    """ Send a message. """
    self._comm.sendMsg(msg)

  def checkMsg(self, expectedMsg, retryCount=None):
    """ Check that the next message is same as expected message. """
    msg = self.recv(retryCount)
    if msg != expectedMsg:
      print >>sys.stderr,'Error with message'
      print >>sys.stderr,'  expected: %s' % expectedMsg
      print >>sys.stderr,'  received: %s' % msg
      # print >>sys.stderr,len(expectedMsg),len(msg)
      raise ValueError

  def convertToExtPlayer(self, team, num):
    """ Returns the external player number for a given player. """
    assert team == self._offenseTeamName or team == self._defenseTeamName,\
      'Invalid team name %s. Valid choices: %s, %s.'\
      %(team, self._offenseTeamName, self._defenseTeamName)
    if team == self._offenseTeamName:
      return self._offenseOrder[num]
    else:
      return self._defenseOrder[num]

  def registerMsgHandler(self,handler,*args,**kwargs):
    '''Register a message handler.

    Handler will be called on a message that matches *args.

    '''
    args = list(args)
    i,_,_ = self._findHandlerInd(args)
    if i < 0:
      self._msgHandlers.append([args,handler])
    else:
      if ('quiet' not in kwargs) or (not kwargs['quiet']):
        print 'Updating handler for %s' % (' '.join(args))
      self._msgHandlers[i] = [args,handler]

  def unregisterMsgHandler(self, *args):
    """ Delete a message handler. """
    i,_,_ = self._findHandlerInd(args)
    assert(i >= 0)
    del self._msgHandlers[i]

  def _findHandlerInd(self, msg):
    """ Find the handler for a particular message. """
    msg = list(msg)
    for i,(partial,handler) in enumerate(self._msgHandlers):
      recPartial = msg[:len(partial)]
      if recPartial == partial:
        return i,len(partial),handler
    return -1,None,None

  def handleMsg(self, msg):
    """ Handle a message using the registered handlers. """
    i,prefixLength,handler = self._findHandlerInd(msg)
    if i < 0:
      print 'Unhandled message:',msg[0:2]
    else:
      handler(msg[prefixLength:])

  def ignoreMsg(self,*args,**kwargs):
    """ Ignore a certain type of message. """
    self.registerMsgHandler(lambda x: None,*args,**kwargs)

  def listenAndProcess(self, retry_count=None):
    """ Gather messages and process them. """
    try:
      msg = self.recv(retry_count)
      assert((msg[0] == '(') and (msg[-1] == ')')),'|%s|' % msg
      msg = self.parseMsg(msg)
      self.handleMsg(msg)
    except TimeoutError:
      pass

  def disconnectPlayer(self, player, player_num, on_offense):
    """Wait on a launched player to disconnect from the server. """
    # print 'Disconnect %s-%d'%(self._offenseTeamName if on_offense
    #                           else self._defenseTeamName, player_num)
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    self.send('(disconnect_player %s %d)'%(team_name, player_num))
    player.kill()

  def waitOnPlayer(self, player_num, on_offense):
    """Wait on a launched player to connect and be reported by the
    server.

    """
    self.send('(look)')
    partial = ['ok','look']
    self._numPlayers = 0
    def f(body):
      del self._connectedPlayers[:]
      for i in xrange(4, len(body)):
        _,team,num = body[i][0][:3]
        if (team, num) not in self._connectedPlayers:
          self._connectedPlayers.append((team,num))
    self.registerMsgHandler(f,*partial,quiet=True)
    team_name = self._offenseTeamName if on_offense else self._defenseTeamName
    while (team_name, str(player_num)) not in self._connectedPlayers:
      self.listenAndProcess()
      self.send('(look)')
    self.ignoreMsg(*partial,quiet=True)

  def checkIfAllPlayersConnected(self):
    """ Returns true if all players are connected. """
    print 'Checking all players are connected'
    self.send('(look)')
    partial = ['ok','look']
    self._numPlayers = 0
    def f(x):
      self._numPlayers = len(x) - 4 # -4 for time, ball, goal_l, and goal_r
      self.send('(look)')
    self.registerMsgHandler(f,*partial,quiet=True)
    while self._numPlayers != self._numOffense + self._numDefense:
      self.listenAndProcess()
    self.ignoreMsg(*partial,quiet=True)

  def startGame(self):
    """ Starts a game of HFO. """
    self.send('(change_mode play_on)')
    self._isPlaying = True

  def printStats(self):
    print 'TotalFrames = %i, AvgFramesPerTrial = %.1f, AvgFramesPerGoal = %.1f'\
      %(self._numFrames,
        self._numFrames / float(self._numTrials) if self._numTrials > 0 else float('nan'),
        self._numGoalFrames / float(self._numGoals) if self._numGoals > 0 else float('nan'))
    print 'Trials             : %i' % self._numTrials
    print 'Goals              : %i' % self._numGoals
    print 'Defense Captured   : %i' % self._numBallsCaptured
    print 'Balls Out of Bounds: %i' % self._numBallsOOB
    print 'Out of Time        : %i' % self._numOutOfTime

  def checkLive(self, necProcesses):
    """Returns true if each of the necessary processes is still alive and
    running.

    """
    for p,name in necProcesses:
      if p.poll() is not None:
        print 'Something necessary closed (%s), exiting' % name
        return False
    return True

  def run(self, necProcesses):
    """ Run the trainer """
    try:
      self.setTeams()
      offense_unums = self._offenseOrder[1: self._numOffense + 1]
      sorted_offense_agent_unums = sorted(self._offenseOrder[1:self._offenseAgents+1])
      defense_unums = self._defenseOrder[: self._numDefense]
      sorted_defense_agent_unums = sorted(self._defenseOrder[:self._defenseAgents])

      # Launch offense
      agent_num = 0
      for player_num in xrange(1, 12):
        if agent_num < self._offenseAgents and player_num == sorted_offense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num
          agent = self.launch_agent(agent_num, sorted_offense_agent_unums[agent_num],
                                    play_offense=True, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = self.launch_npc(player_num, play_offense=True)
          if player_num in offense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'offense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=True)

      # Launch defense
      agent_num = 0
      for player_num in xrange(1, 12):
        if agent_num < self._defenseAgents and player_num == sorted_defense_agent_unums[agent_num]:
          port = self._agentServerPort + agent_num + self._offenseAgents
          agent = self.launch_agent(agent_num, sorted_defense_agent_unums[agent_num],
                                    play_offense=False, port=port)
          self._agentPopen.append(agent)
          necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
          agent_num += 1
        else:
          player = self.launch_npc(player_num, play_offense=False)
          if player_num in defense_unums:
            self._npcPopen.append(player)
            necProcesses.append([player, 'defense_npc_' + str(player_num)])
          else:
            self.disconnectPlayer(player, player_num, on_offense=False)

      self.checkIfAllPlayersConnected()

      if self._numAgents > 0:
        print 'Agents awaiting your connections'
        necOff = set([(self._offenseTeamName,str(x)) for x in sorted_offense_agent_unums])
        necDef = set([(self._defenseTeamName,str(x)) for x in sorted_defense_agent_unums])
        necAgents = necOff.union(necDef)
        while self.checkLive(necProcesses) and self._agentReady != necAgents:
          self.listenAndProcess()

      # Broadcast the HFO configuration
      offense_nums = ' '.join([str(self.convertToExtPlayer(self._offenseTeamName, i))
                               for i in xrange(1, self._numOffense + 1)])
      defense_nums = ' '.join([str(self.convertToExtPlayer(self._defenseTeamName, i))
                               for i in xrange(self._numDefense)])
      self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
                  ' num_defense %d offense_nums %s defense_nums %s)'
                %(self._offenseTeamName, self._defenseTeamName,
                  self._numOffense, self._numDefense,
                  offense_nums, defense_nums))
      print 'Starting game'
      self.startGame()
      while self.checkLive(necProcesses) and not self._done:
        prevFrame = self._frame
        self.listenAndProcess()
    except TimeoutError:
      print 'Haven\'t heard from the server for too long, Exiting'
    except (KeyboardInterrupt, DoneError):
      print 'Finished'
    finally:
      try:
        self._comm.sendMsg('(bye)')
      except:
        pass
      for p in self._agentPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      for p in self._npcPopen:
        try:
          p.terminate()
          time.sleep(0.1)
          p.kill()
        except:
          pass
      self._comm.close()
      self.printStats()
Exemplo n.º 5
0
class Trainer(object):
    """ Trainer is responsible for setting up the players and game.
  """
    def __init__(self,
                 args,
                 rng=numpy.random.RandomState(),
                 server_port=6001,
                 coach_port=6002):
        self._rng = rng  # The Random Number Generator
        self._serverPort = server_port  # The port the server is listening on
        self._coachPort = coach_port  # The coach port to talk with the server
        self._logDir = args.logDir  # Directory to store logs
        self._record = args.record  # Record states + actions
        self._numOffense = args.offenseAgents + args.offenseNPCs  # Number offensive players
        self._numDefense = args.defenseAgents + args.defenseNPCs  # Number defensive players
        self._maxTrials = args.numTrials  # Maximum number of trials to play
        self._maxFrames = args.numFrames  # Maximum number of frames to play
        self._maxFramesPerTrial = args.maxFramesPerTrial
        # =============== FIELD DIMENSIONS =============== #
        self.NUM_FRAMES_TO_HOLD = 2  # Hold ball this many frames to capture
        self.HOLD_FACTOR = 1.5  # Gain to calculate ball control
        self.PITCH_WIDTH = 68.0  # Width of the field
        self.PITCH_LENGTH = 105.0  # Length of field in long-direction
        self.UNTOUCHED_LENGTH = 100  # Trial will end if ball untouched for this long
        # allowedBallX, allowedBallY defines the usable area of the playfield
        self._allowedBallX = numpy.array([-0.1, 0.5 * self.PITCH_LENGTH])
        self._allowedBallY = numpy.array(
            [-0.5 * self.PITCH_WIDTH, 0.5 * self.PITCH_WIDTH])
        # =============== COUNTERS =============== #
        self._numFrames = 0  # Number of frames seen in HFO trials
        self._frame = 0  # Current frame id
        self._lastTrialStart = -1  # Frame Id in which the last trial started
        self._lastFrameBallTouched = -1  # Frame Id in which ball was last touched
        # =============== TRIAL RESULTS =============== #
        self._numTrials = 0  # Total number of HFO trials
        self._numGoals = 0  # Trials in which the offense scored a goal
        self._numBallsCaptured = 0  # Trials in which defense captured the ball
        self._numBallsOOB = 0  # Trials in which ball went out of bounds
        self._numOutOfTime = 0  # Trials that ran out of time
        # =============== AGENT =============== #
        self._numAgents = args.offenseAgents + args.defenseAgents
        self._offenseAgents = args.offenseAgents
        self._defenseAgents = args.defenseAgents
        self._agentTeams = []  # Names of the teams the agents are playing for
        self._agentNumInt = []  # List of agents internal team numbers
        self._agentNumExt = []  # List of agents external team numbers
        self._agentServerPort = args.port  # Base Port for agent's server
        self._agentOnBall = args.agent_on_ball  # If true, agent starts with the ball
        # =============== MISC =============== #
        self._offenseTeamName = ''  # Name of the offensive team
        self._defenseTeamName = ''  # Name of the defensive team
        self._playerPositions = numpy.zeros(
            (11, 2, 2))  # Positions of the players
        self._ballPosition = numpy.zeros(2)  # Position of the ball
        self._ballHeld = numpy.zeros((11, 2))  # Track player holding the ball
        self._teams = []  # Team indexes for offensive and defensive teams
        self._SP = {
        }  # Sever Parameters. Recieved when connecting to the server.
        self._isPlaying = False  # Is a game being played?
        self._teamHoldingBall = None  # Team currently in control of the ball
        self._playerHoldingBall = None  # Player current in control of ball
        self._agentPopen = []  # Agent's processes
        self.initMsgHandlers()

    def launch_agent(self, agent_num, play_offense, port):
        """Launch the learning agent using the start_agent.sh script and
    return a DummyPopen for the process.

    """
        print '[Trainer] Launching Agent', str(agent_num)
        if play_offense:
            assert self._numOffense > 0
            team_name = self._offenseTeamName
            self._agentTeams.append(team_name)
            # First offense number is reserved for inactive offensive goalie
            internal_player_num = agent_num + 1
            self._agentNumInt.append(internal_player_num)
            numTeammates = self._numOffense - 1
            numOpponents = self._numDefense
        else:
            assert self._numDefense > 0
            team_name = self._defenseTeamName
            self._agentTeams.append(team_name)
            internal_player_num = agent_num
            self._agentNumInt.append(internal_player_num)
            numTeammates = self._numDefense - 1
            numOpponents = self._numOffense
        ext_num = self.convertToExtPlayer(team_name, internal_player_num)
        self._agentNumExt.append(ext_num)
        binary_dir = os.path.dirname(os.path.realpath(__file__))
        agentCmd = 'start_agent.sh -t %s -u %i -p %i -P %i --log-dir %s'\
                   ' --numTeammates %i --numOpponents %i'\
                   ' --playingOffense %i --serverPort %i'\
                   %(team_name, ext_num, self._serverPort,
                     self._coachPort, self._logDir, numTeammates,
                     numOpponents, play_offense, port)
        if self._record:
            agentCmd += ' --record'
        agentCmd = os.path.join(binary_dir, agentCmd)
        agentCmd = agentCmd.split(' ')
        # Ignore stderr because librcsc continually prints to it
        kwargs = {}  #{'stderr':open('/dev/null','w')}
        p = subprocess.Popen(agentCmd, **kwargs)
        p.wait()
        pid_file = os.path.join(self._logDir, 'start%i' % p.pid)
        print '[Trainer] Parsing agent\'s pid from file:', pid_file
        assert os.path.isfile(pid_file)
        with open(pid_file, 'r') as f:
            output = f.read()
        pid = int(re.findall('PID: (\d+)', output)[0])
        return DummyPopen(pid)

    def getDefensiveRoster(self, team_name):
        """Returns a list of player numbers on a given team that are thought
    to prefer defense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
        if team_name == 'Borregos':
            return [9, 10, 8, 11, 7, 4, 6, 2, 3, 5]
        elif team_name == 'WrightEagle':
            return [5, 2, 8, 9, 10, 6, 3, 11, 4, 7]
        else:
            return [2, 3, 4, 5, 6, 7, 8, 11, 9, 10]

    def getOffensiveRoster(self, team_name):
        """Returns a list of player numbers on a given team that are thought
    to prefer offense. This map is not set in stone as the players on
    some teams can adapt and change their roles.

    """
        if team_name == 'Borregos':
            return [2, 4, 6, 5, 3, 7, 9, 10, 8, 11]
        elif team_name == 'WrightEagle':
            return [11, 4, 7, 3, 6, 10, 8, 9, 2, 5]
        else:
            return [11, 7, 8, 9, 10, 6, 3, 2, 4, 5]

    def setTeams(self):
        """ Sets the offensive and defensive teams and player rosters. """
        self._offenseTeamInd = 0
        self._offenseTeamName = self._teams[self._offenseTeamInd]
        self._defenseTeamName = self._teams[1 - self._offenseTeamInd]
        offensive_roster = self.getOffensiveRoster(self._offenseTeamName)
        defensive_roster = self.getDefensiveRoster(self._defenseTeamName)
        self._offenseOrder = [1] + offensive_roster  # 1 for goalie
        self._defenseOrder = [1] + defensive_roster  # 1 for goalie

    def teamToInd(self, team_name):
        """ Returns the index of a given team. """
        return self._teams.index(team_name)

    def parseMsg(self, msg):
        """ Parse a message """
        assert (msg[0] == '(')
        res, ind = self.__parseMsg(msg, 1)
        assert (ind == len(msg)), msg
        return res

    def __parseMsg(self, msg, ind):
        """ Recursively parse a message. """
        res = []
        while True:
            if msg[ind] == '"':
                ind += 1
                startInd = ind
                while msg[ind] != '"':
                    ind += 1
                res.append(msg[startInd:ind])
                ind += 1
            elif msg[ind] == '(':
                inner, ind = self.__parseMsg(msg, ind + 1)
                res.append(inner)
            elif msg[ind] == ' ':
                ind += 1
            elif msg[ind] == ')':
                return res, ind + 1
            else:
                startInd = ind
                while msg[ind] not in ' ()':
                    ind += 1
                res.append(msg[startInd:ind])

    def initComm(self):
        """ Initialize communication to server. """
        self._comm = ClientCommunicator(port=self._coachPort)
        self.send('(init (version 8.0))')
        self.checkMsg('(init ok)', retryCount=5)
        # self.send('(eye on)')
        self.send('(ear on)')

    def _hear(self, body):
        """ Handle a hear message. """
        timestep, playerInfo, msg = body
        if len(playerInfo) != 3:
            return
        _, team, player = playerInfo
        try:
            length = int(msg[0])
        except:
            return
        msg = msg[1:length + 1]
        if msg == 'START':
            if self._isPlaying:
                print '[Trainer] Already playing, ignoring message'
            else:
                self.startGame()
        elif msg == 'DONE':
            raise DoneError
        else:
            print '[Trainer] Unhandled message from agent: %s' % msg

    def initMsgHandlers(self):
        """ Create handlers for different messages. """
        self._msgHandlers = []
        self.ignoreMsg('player_param')
        self.ignoreMsg('player_type')
        self.ignoreMsg('ok', 'change_mode')
        self.ignoreMsg('ok', 'ear')
        self.ignoreMsg('ok', 'move')
        self.ignoreMsg('ok', 'recover')
        self.ignoreMsg('ok', 'say')
        self.registerMsgHandler(self._handleSP, 'server_param')
        self.registerMsgHandler(self._hear, 'hear')

    def recv(self, retryCount=None):
        """ Recieve a message. Retry a specified number of times. """
        return self._comm.recvMsg(retryCount=retryCount).strip()

    def send(self, msg):
        """ Send a message. """
        self._comm.sendMsg(msg)

    def checkMsg(self, expectedMsg, retryCount=None):
        """ Check that the next message is same as expected message. """
        msg = self.recv(retryCount)
        if msg != expectedMsg:
            print >> sys.stderr, '[Trainer] Error with message'
            print >> sys.stderr, '  expected: %s' % expectedMsg
            print >> sys.stderr, '  received: %s' % msg
            # print >>sys.stderr,len(expectedMsg),len(msg)
            raise ValueError

    def extractPoint(self, msg):
        """ Extract a point from the provided message. """
        return numpy.array(map(float, msg[:2]))

    def convertToExtPlayer(self, team, num):
        """ Returns the external player number for a given player. """
        assert team == self._offenseTeamName or team == self._defenseTeamName,\
          'Invalid team name %s. Valid choices: %s, %s.'\
          %(team, self._offenseTeamName, self._defenseTeamName)
        if team == self._offenseTeamName:
            return self._offenseOrder[num]
        else:
            return self._defenseOrder[num]

    def convertFromExtPlayer(self, team, num):
        """ Maps external player number to internal player number. """
        if team == self._offenseTeamName:
            return self._offenseOrder.index(num)
        else:
            return self._defenseOrder.index(num)

    def seeGlobal(self, body):
        """Send a look message to extract global information on ball and
    player positions.

    """
        self.send('(look)')
        self._frame = int(body[0])
        for obj in body[1:]:
            objType = obj[0]
            objData = obj[1:]
            if objType[0] == 'g':
                continue
            elif objType[0] == 'b':
                self._ballPosition = self.extractPoint(objData)
            elif objType[0] == 'p':
                teamName = objType[1]
                team = self.teamToInd(teamName)
                playerNum = self.convertFromExtPlayer(teamName,
                                                      int(objType[2]))
                self._playerPositions[playerNum, :,
                                      team] = self.extractPoint(objData)

    def registerMsgHandler(self, handler, *args, **kwargs):
        '''Register a message handler.

    Handler will be called on a message that matches *args.

    '''
        args = list(args)
        i, _, _ = self._findHandlerInd(args)
        if i < 0:
            self._msgHandlers.append([args, handler])
        else:
            if ('quiet' not in kwargs) or (not kwargs['quiet']):
                print '[Trainer] Updating handler for %s' % (' '.join(args))
            self._msgHandlers[i] = [args, handler]

    def unregisterMsgHandler(self, *args):
        """ Delete a message handler. """
        i, _, _ = self._findHandlerInd(args)
        assert (i >= 0)
        del self._msgHandlers[i]

    def _findHandlerInd(self, msg):
        """ Find the handler for a particular message. """
        msg = list(msg)
        for i, (partial, handler) in enumerate(self._msgHandlers):
            recPartial = msg[:len(partial)]
            if recPartial == partial:
                return i, len(partial), handler
        return -1, None, None

    def handleMsg(self, msg):
        """ Handle a message using the registered handlers. """
        i, prefixLength, handler = self._findHandlerInd(msg)
        if i < 0:
            print '[Trainer] Unhandled message:', msg[0:2]
        else:
            handler(msg[prefixLength:])

    def ignoreMsg(self, *args, **kwargs):
        """ Ignore a certain type of message. """
        self.registerMsgHandler(lambda x: None, *args, **kwargs)

    def _handleSP(self, body):
        """ Handler for the sever params message. """
        for param in body:
            try:
                val = int(param[1])
            except:
                try:
                    val = float(param[1])
                except:
                    val = param[1]
            self._SP[param[0]] = val

    def listenAndProcess(self):
        """ Gather messages and process them. """
        msg = self.recv()
        assert ((msg[0] == '(') and (msg[-1] == ')')), '|%s|' % msg
        msg = self.parseMsg(msg)
        self.handleMsg(msg)

    def _readTeamNames(self, body):
        """ Read the names of each of the teams. """
        self._teams = []
        for _, _, team in body:
            self._teams.append(team)
        time.sleep(0.1)
        self.send('(team_names)')

    def waitOnTeam(self, first):
        """Wait on a given team. First indicates if this is the first team
    connected or the second.

    """
        self.send('(team_names)')
        partial = ['ok', 'team_names']
        self.registerMsgHandler(self._readTeamNames, *partial, quiet=True)
        while len(self._teams) < (1 if first else 2):
            self.listenAndProcess()
        #self.unregisterMsgHandler(*partial)
        self.ignoreMsg(*partial, quiet=True)

    def checkIfAllPlayersConnected(self):
        """ Returns true if all players are connected. """
        self.send('(look)')
        partial = ['ok', 'look']
        self._numPlayers = 0

        def f(x):
            self._numPlayers = len(
                x) - 4  # -4 for time, ball, goal_l, and goal_r
            self.send('(look)')

        self.registerMsgHandler(f, *partial)
        while self._numPlayers != 2 * 11:
            self.listenAndProcess()
        self.ignoreMsg(*partial, quiet=True)

    def startGame(self):
        """ Starts a game of HFO. """
        self.reset()
        self.registerMsgHandler(self.seeGlobal, 'see_global')
        self.registerMsgHandler(self.seeGlobal, 'ok', 'look', quiet=True)
        #self.registerMsgHandler(self.checkBall,'ok','check_ball')
        self.send('(look)')
        self._isPlaying = True

    def calcBallHolder(self):
        '''Calculates the ball holder, returns results in teamInd, playerInd. '''
        totalHeld = 0
        for team in self._teams:
            for i in range(11):
                pos = self._playerPositions[i, :, self.teamToInd(team)]
                distBound = self._SP['kickable_margin'] + self._SP['player_size'] \
                            + self._SP['ball_size']
                distBound *= self.HOLD_FACTOR
                if numpy.linalg.norm(self._ballPosition - pos) < distBound:
                    self._ballHeld[i, self.teamToInd(team)] += 1
                    totalHeld += 1
                else:
                    self._ballHeld[i, self.teamToInd(team)] = 0
        # If multiple players are close to the ball, no-one is holding
        if totalHeld > 1:
            self._ballHeld[:, :] = 0
        inds = numpy.transpose(
            (self._ballHeld >= self.NUM_FRAMES_TO_HOLD).nonzero())
        assert (len(inds) <= 1)
        if len(inds) == 1:
            return inds[0, 1], inds[0, 0]
        else:
            return None, None

    def isGoal(self):
        """ Returns true if a goal has been scored. """
        return (self._ballPosition[0] > self._allowedBallX[1]) \
          and (numpy.abs(self._ballPosition[1]) <= 0.5 * self._SP['goal_width'])

    def isOOB(self):
        """ Returns true if the ball is out of bounds. """
        return self._ballPosition[0] < self._allowedBallX[0] \
          or self._ballPosition[0] > self._allowedBallX[1] \
          or self._ballPosition[1] < self._allowedBallY[0] \
          or self._ballPosition[1] > self._allowedBallY[1]

    def isCaptured(self):
        """ Returns true if the ball is captured by defense. """
        return self._teamHoldingBall not in [None, self._offenseTeamInd]

    def isOOT(self):
        """ Returns true if the trial has run out of time. """
        return self._frame - self._lastFrameBallTouched > self.UNTOUCHED_LENGTH \
          or (self._maxFramesPerTrial > 0 and self._frame -
              self._lastTrialStart > self._maxFramesPerTrial)

    def movePlayer(self, team, internal_num, pos, convertToExt=True):
        """ Move a player to a specified position.
    Args:
      team: the team name of the player
      interal_num: the player's internal number
      pos: position to move player to
      convertToExt: convert interal player num to external
    """
        num = self.convertToExtPlayer(team, internal_num) if convertToExt \
              else internal_num
        self.send('(move (player %s %i) %f %f)' % (team, num, pos[0], pos[1]))

    def moveBall(self, pos):
        """ Moves the ball to a specified x,y position. """
        self.send('(move (ball) %f %f 0.0 0.0 0.0)' % tuple(pos))

    def randomPointInBounds(self, xBounds=None, yBounds=None):
        """Returns a random point inside of the box defined by xBounds,
    yBounds. Where xBounds=[x_min, x_max] and yBounds=[y_min,
    y_max]. Defaults to the xy-bounds of the playable HFO area.

    """
        if xBounds is None:
            xBounds = self.allowedBallX
        if yBounds is None:
            yBounds = self.allowedBallY
        pos = numpy.zeros(2)
        bounds = [xBounds, yBounds]
        for i in range(2):
            pos[i] = self._rng.rand() * (bounds[i][1] -
                                         bounds[i][0]) + bounds[i][0]
        return pos

    def boundPoint(self, pos):
        """Ensures a point is within the minimum and maximum bounds of the
    HFO playing area.

    """
        pos[0] = min(max(pos[0], self._allowedBallX[0]), self._allowedBallX[1])
        pos[1] = min(max(pos[1], self._allowedBallY[0]), self._allowedBallY[1])
        return pos

    def reset(self):
        """ Resets the HFO domain by moving the ball and players. """
        self.resetBallPosition()
        self.resetPlayerPositions()
        self.send('(recover)')
        self.send('(change_mode play_on)')
        # self.send('(say RESET)')

    def resetBallPosition(self):
        """Reset the position of the ball for a new HFO trial. """
        self._ballPosition = self.boundPoint(
            self.randomPointInBounds(
                .2 * self._allowedBallX + .05 * self.PITCH_LENGTH,
                .8 * self._allowedBallY))
        self.moveBall(self._ballPosition)

    def getOffensiveResetPosition(self):
        """ Returns a random position for an offensive player. """
        offsets = [
            [-1, -1],
            [-1, 1],
            [1, 1],
            [1, -1],
            [0, 2],
            [0, -2],
            [-2, -2],
            [-2, 2],
            [2, 2],
            [2, -2],
        ]
        offset = offsets[self._rng.randint(len(offsets))]
        offset_from_ball = 0.1 * self.PITCH_LENGTH * self._rng.rand(2) + \
                           0.1 * self.PITCH_LENGTH * numpy.array(offset)
        return self.boundPoint(self._ballPosition + offset_from_ball)
        # return self._ballPosition

    def getDefensiveResetPosition(self):
        """ Returns a random position for a defensive player. """
        return self.boundPoint(
            self.randomPointInBounds([
                0.5 * 0.5 * self.PITCH_LENGTH, 0.75 * 0.5 * self.PITCH_LENGTH
            ], 0.8 * self._allowedBallY))

    def resetPlayerPositions(self):
        """Reset the positions of the players. This is called after a trial
    ends to setup for the next trial.

    """
        # Always Move the offensive goalie to the left goal
        self.movePlayer(self._offenseTeamName, 0,
                        [-0.5 * self.PITCH_LENGTH, 0])
        # Move the rest of the offense
        for i in xrange(1, self._numOffense + 1):
            self.movePlayer(self._offenseTeamName, i,
                            self.getOffensiveResetPosition())
        # Move the agent to the ball
        if self._agentOnBall and self._offenseAgents > 0:
            self.movePlayer(self._offenseTeamName, self._agentNumInt[0],
                            self._ballPosition)
        # Move the defensive goalie
        if self._numDefense > 0:
            self.movePlayer(self._defenseTeamName, 0,
                            [0.5 * self.PITCH_LENGTH, 0])
        # Move the rest of the defense
        for i in xrange(1, self._numDefense):
            self.movePlayer(self._defenseTeamName, i,
                            self.getDefensiveResetPosition())

    def removeNonHFOPlayers(self):
        """Removes players that aren't involved in HFO game.

    The players whose numbers are greater than numOffense/numDefense
    are sent to left-field.

    """
        offensive_agent_numbers = self._agentNumInt[:self._offenseAgents]
        defensive_agent_numbers = self._agentNumInt[self._offenseAgents:]
        for i in xrange(self._numOffense + 1, 11):
            if i not in offensive_agent_numbers:
                self.movePlayer(self._offenseTeamName, i,
                                [-0.25 * self.PITCH_LENGTH, 0])
        for i in xrange(self._numDefense, 11):
            if i not in defensive_agent_numbers:
                self.movePlayer(self._defenseTeamName, i,
                                [-0.25 * self.PITCH_LENGTH, 0])

    def step(self):
        """ Takes a simulated step. """
        # self.send('(check_ball)')
        self.removeNonHFOPlayers()
        self._teamHoldingBall, self._playerHoldingBall = self.calcBallHolder()
        if self._teamHoldingBall is not None:
            self._lastFrameBallTouched = self._frame
        if self.trialOver():
            self.updateResults()
            self._lastFrameBallTouched = self._frame
            self.reset()

    def updateResults(self):
        """ Updates the various members after a trial has ended. """
        if self.isGoal():
            self._numGoals += 1
            result = 'Goal'
            self.send('(say GOAL)')
        elif self.isOOB():
            self._numBallsOOB += 1
            result = 'Out of Bounds'
            self.send('(say OUT_OF_BOUNDS)')
        elif self.isCaptured():
            self._numBallsCaptured += 1
            result = 'Defense Captured'
            self.send('(say CAPTURED_BY_DEFENSE)')
        elif self.isOOT():
            self._numOutOfTime += 1
            result = 'Ball untouched for too long'
            self.send('(say OUT_OF_TIME)')
        else:
            print '[Trainer] Error: Unable to detect reason for End of Trial!'
            sys.exit(1)
        print '[Trainer] EndOfTrial: %2i /%2i %5i %s'%\
          (self._numGoals, self._numTrials, self._frame, result)
        self._numTrials += 1
        self._numFrames += self._frame - self._lastTrialStart
        self._lastTrialStart = self._frame
        if (self._maxTrials > 0) and (self._numTrials >= self._maxTrials):
            raise DoneError
        if (self._maxFrames > 0) and (self._numFrames >= self._maxFrames):
            raise DoneError

    def trialOver(self):
        """Returns true if the trial has ended for one of the following
    reasons: Goal scored, out of bounds, captured by defense, or
    untouched for too long.

    """
        # The trial is still being setup, it cannot be over.
        if self._frame - self._lastTrialStart < 5:
            return False
        return self.isGoal() or self.isOOB() or self.isCaptured(
        ) or self.isOOT()

    def printStats(self):
        print 'Num frames in completed trials : %i' % (self._numFrames)
        print 'Trials             : %i' % self._numTrials
        print 'Goals              : %i' % self._numGoals
        print 'Defense Captured   : %i' % self._numBallsCaptured
        print 'Balls Out of Bounds: %i' % self._numBallsOOB
        print 'Out of Time        : %i' % self._numOutOfTime

    def checkLive(self, necProcesses):
        """Returns true if each of the necessary processes is still alive and
    running.

    """
        for p, name in necProcesses:
            if p.poll() is not None:
                print '[Trainer] Something necessary closed (%s), exiting' % name
                return False
        return True

    def run(self, necProcesses):
        """ Run the trainer """
        try:
            for agent_num in xrange(self._offenseAgents):
                port = self._agentServerPort + agent_num
                agent = self.launch_agent(agent_num,
                                          play_offense=True,
                                          port=port)
                self._agentPopen.append(agent)
                necProcesses.append([agent, 'offense_agent_' + str(agent_num)])
            for agent_num in xrange(self._defenseAgents):
                port = self._agentServerPort + agent_num + self._offenseAgents
                agent = self.launch_agent(agent_num,
                                          play_offense=False,
                                          port=port)
                self._agentPopen.append(agent)
                necProcesses.append([agent, 'defense_agent_' + str(agent_num)])
            # Broadcast the HFO configuration
            offense_nums = ' '.join([
                str(self.convertToExtPlayer(self._offenseTeamName, i))
                for i in xrange(1, self._numOffense + 1)
            ])
            defense_nums = ' '.join([
                str(self.convertToExtPlayer(self._defenseTeamName, i))
                for i in xrange(self._numDefense)
            ])
            self.send('(say HFO_SETUP offense_name %s defense_name %s num_offense %d'\
                        ' num_defense %d offense_nums %s defense_nums %s)'
                      %(self._offenseTeamName, self._defenseTeamName,
                        self._numOffense, self._numDefense,
                        offense_nums, defense_nums))
            self.startGame()
            while self.checkLive(necProcesses):
                prevFrame = self._frame
                self.listenAndProcess()
                if self._frame != prevFrame:
                    self.step()
        except TimeoutError:
            print '[Trainer] Haven\'t heard from the server for too long, Exiting'
        except (KeyboardInterrupt, DoneError):
            print '[Trainer] Exiting'
        finally:
            for p in self._agentPopen:
                p.send_signal(SIGINT)
            try:
                self._comm.sendMsg('(bye)')
            except:
                pass
            self._comm.close()
            self.printStats()