Beispiel #1
0
  def __init__(self, db_name='fishtest_new', clop_socket=None):
    # MongoDB server is assumed to be on the same machine, if not user should use
    # ssh with port forwarding to access the remote host.
    self.conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost')
    self.db = self.conn[db_name]
    self.userdb = UserDb(self.db)
    self.clopdb = ClopDb(self.db, clop_socket)
    self.actiondb = ActionDb(self.db)
    self.runs = self.db['runs']

    self.chunk_size = 1000
Beispiel #2
0
class RunDb:
  def __init__(self, db_name='fishtest_new', clop_socket=None):
    # MongoDB server is assumed to be on the same machine, if not user should use
    # ssh with port forwarding to access the remote host.
    self.conn = MongoClient(os.getenv('FISHTEST_HOST') or 'localhost')
    self.db = self.conn[db_name]
    self.userdb = UserDb(self.db)
    self.clopdb = ClopDb(self.db, clop_socket)
    self.actiondb = ActionDb(self.db)
    self.runs = self.db['runs']

    self.chunk_size = 1000

  def generate_tasks(self, num_games):
    tasks = []
    remaining = num_games
    while remaining > 0:
      task_size = min(self.chunk_size, remaining)
      tasks.append({
        'num_games': task_size,
        'pending': True,
        'active': False,
      })
      remaining -= task_size
    return tasks

  def new_run(self, base_tag, new_tag, num_games, tc, book, book_depth, threads, base_options, new_options,
              info='',
              resolved_base='',
              resolved_new='',
              msg_base='',
              msg_new='',
              base_signature='',
              new_signature='',
              regression_test=False,
              start_time=None,
              sprt=None,
              clop=None,
              username=None,
              tests_repo=None,
              priority=0):
    if start_time == None:
      start_time = datetime.utcnow()

    run_args = {
      'base_tag': base_tag,
      'new_tag': new_tag,
      'num_games': num_games,
      'tc': tc,
      'book': book,
      'book_depth': book_depth,
      'threads': threads,
      'regression_test': regression_test,
      'resolved_base': resolved_base,
      'resolved_new': resolved_new,
      'msg_base': msg_base,
      'msg_new': msg_new,
      'base_options': base_options,
      'new_options': new_options,
      'info': info,
      'base_signature': base_signature,
      'new_signature': new_signature,
      'username': username,
      'tests_repo': tests_repo,
      'priority': priority,
    }

    if sprt != None:
      run_args['sprt'] = sprt

    if clop != None:
      run_args['clop'] = clop

    new_run = {
      'args': run_args,
      'start_time': start_time,
      # Will be filled in by tasks, indexed by task-id
      'tasks': self.generate_tasks(num_games),
      # Aggregated results
      'results': { 'wins': 0, 'losses': 0, 'draws': 0 },
      'results_stale': False,
      'finished': False,
      'approved': False,
      'approver': '',
    }

    # Check for an existing approval matching the git commit SHAs
    def get_approval(sha):
      q = { '$or': [{ 'args.resolved_base': sha }, { 'args.resolved_new': sha }], 'approved': True }
      return self.runs.find_one(q)
    base_approval = get_approval(resolved_base)
    new_approval = get_approval(resolved_new)
    if base_approval != None and new_approval != None:
      new_run['approved'] = True
      new_run['approver'] = new_approval['approver']

    return self.runs.insert(new_run)

  def get_machines(self):
    machines = []
    for run in self.runs.find({'tasks': {'$elemMatch': {'active': True}}}):
      for task in run['tasks']:
        if task['active']:
          machine = copy.copy(task['worker_info'])
          machine['last_updated'] = task.get('last_updated', None)
          machine['run'] = run
          machine['nps'] = task.get('nps', 0)
          # TODO(glinscott): Temporary - remove once worker version >= 41
          if not isinstance(machine['uname'], basestring):
            machine['uname'] = machine['uname'][0] + machine['uname'][2]
          machines.append(machine)
    return machines

  def get_run(self, id):
    return self.runs.find_one({'_id': ObjectId(id)})

  def get_run_to_build(self):
    return self.runs.find_one({'binaries_url': {'$exists': False}, 'finished': False, 'deleted': {'$exists': False}})

  def get_runs(self, skip=0, limit=0):
    runs = []
    for run in self.runs.find(skip=skip,
                              limit=limit,
                              sort=[('last_updated', DESCENDING), ('start_time', DESCENDING)]):
      runs.append(run)
    return runs

  def get_clop_exclusion_list(self, minimum):
    exclusion_list = []
    for run in self.runs.find({'args.clop': {'$exists': True}, 'finished': False, 'deleted': {'$exists': False}}):
      total_games = 0
      available_games = 0
      for game in self.clopdb.get_games(run['_id']):
        total_games += 1
        if len(game['task_id']) == 0:
          available_games += 1

      if total_games > 0 and available_games < minimum:
        exclusion_list.append(run['_id'])
    return exclusion_list

  def get_results(self, run):
    if not run['results_stale']:
      return run['results']

    results = { 'wins': 0, 'losses': 0, 'draws': 0, 'crashes': 0 }
    for task in run['tasks']:
      if 'stats' in task:
        stats = task['stats']
        results['wins'] += stats['wins']
        results['losses'] += stats['losses']
        results['draws'] += stats['draws']
        results['crashes'] += stats['crashes']

    if 'sprt' in run['args'] and 'state' in run['args']['sprt']:
      results['sprt'] = run['args']['sprt']['state']

    run['results_stale'] = False
    run['results'] = results
    self.runs.save(run)

    return results

  def request_task(self, worker_info):

    # Build list of CLOP runs that are already almost full
    max_threads = int(worker_info['concurrency'])
    if worker_info['username'] == 'glinscott':
      exclusion_list = self.get_clop_exclusion_list(5 + max_threads)
    else:
      exclusion_list = self.get_clop_exclusion_list(1000000)

    # Does this worker have a task already?  If so, just hand that back
    existing_run = self.runs.find_one({'tasks': {'$elemMatch': {'active': True, 'worker_info': worker_info}}})
    if existing_run != None and existing_run['_id'] not in exclusion_list:
      for task_id, task in enumerate(existing_run['tasks']):
        if task['active'] and task['worker_info'] == worker_info:
          if task['pending']:
            return {'run': existing_run, 'task_id': task_id}
          else:
            # Don't hand back tasks that have been marked as no longer pending
            task['active'] = False
            self.runs.save(existing_run)

    # We need to allocate a new task, but first check we don't have the same
    # machine already running because multiple connections are not allowed.
    remote_addr = worker_info['remote_addr']
    machines = self.get_machines()
    connections = sum([int(m.get('remote_addr','') == remote_addr) for m in machines])

    # Allow a few connections, for multiple computers on same IP
    if connections > 3:
      return {'task_waiting': False}

    # Ok, we get a new task that does not require more threads than available concurrency
    q = {
      'new': True,
      'query': { '$and': [ {'tasks': {'$elemMatch': {'active': False, 'pending': True}}},
                           {'args.threads': { '$lte': max_threads }},
                           {'_id': { '$nin': exclusion_list}},
                           {'approved': True}]},
      'sort': [('args.priority', DESCENDING), ('_id', ASCENDING)],
      'update': {
        '$set': {
          'tasks.$.active': True,
          'tasks.$.last_updated': datetime.utcnow(),
          'tasks.$.worker_info': worker_info,
        }
      }
    }

    run = self.runs.find_and_modify(**q)
    if run == None:
      return {'task_waiting': False}

    # Find the task we have just activated: the one with the highest 'last_updated'
    latest_time = datetime.min
    for idx, task in enumerate(run['tasks']):
      if 'last_updated' in task and task['last_updated'] > latest_time:
        latest_time = task['last_updated']
        task_id = idx

    # Lower priority of long running tests
    if task_id > 46 and 'sprt' in run['args'] and run['args']['priority'] == 0:
      run['args']['priority'] = -1
      self.runs.save(run)

    return {'run': run, 'task_id': task_id}

  def update_task(self, run_id, task_id, stats, nps, clop):
    run = self.get_run(run_id)
    if task_id >= len(run['tasks']):
      return {'task_alive': False}

    task = run['tasks'][task_id]
    if not task['active'] or not task['pending']:
      return {'task_alive': False}

    # Guard against incorrect results
    num_games = stats['wins'] + stats['losses'] + stats['draws']
    if 'stats' in task and num_games < task['stats']['wins'] + task['stats']['losses'] + task['stats']['draws']:
      return {'task_alive': False}

    task['stats'] = stats
    task['nps'] = nps
    if num_games >= task['num_games']:
      task['active'] = False
      task['pending'] = False

    update_time = datetime.utcnow()
    task['last_updated'] = update_time
    run['last_updated'] = update_time
    run['results_stale'] = True
    self.runs.save(run)

    # Check if SPRT stopping is enabled
    if 'sprt' in run['args']:
      sprt = run['args']['sprt']
      sprt_stats = stat_util.SPRT(self.get_results(run),
                                  elo0=sprt['elo0'],
                                  alpha=sprt['alpha'],
                                  elo1=sprt['elo1'],
                                  beta=sprt['beta'],
                                  drawelo=sprt['drawelo'])
      if sprt_stats['finished']:
        run['args']['sprt']['state'] = sprt_stats['state']
        self.runs.save(run)

        self.stop_run(run_id)

    # Update clop results
    if 'clop' in run['args'] and len(clop['game_id']) > 0:
      self.clopdb.write_result(clop['game_id'], clop['game_result'])
      if not task['active']:
        self.clopdb.stop_games(run_id, task_id)

    return {'task_alive': task['active']}

  def failed_task(self, run_id, task_id):
    run = self.get_run(run_id)
    if task_id >= len(run['tasks']):
      return {'task_alive': False}

    task = run['tasks'][task_id]
    if not task['active'] or not task['pending']:
      return {'task_alive': False}

    # Mark the task as inactive: it will be rescheduled
    task['active'] = False
    self.runs.save(run)

    if 'clop' in run['args']:
      self.clopdb.stop_games(run_id, task_id)

    return {}

  def stop_run(self, run_id):
    run = self.get_run(run_id)
    for idx, task in enumerate(run['tasks']):
      is_active = task['active']
      task['active'] = False
      task['pending'] = False
      if 'stats' not in task and not is_active:
        # Truncate the empty tasks
        del run['tasks'][idx:]
        break
    self.runs.save(run)

    return {}

  def approve_run(self, run_id, approver):
    run = self.get_run(run_id)
    # Can't self approve
    if run['args']['username'] == approver:
      return False

    run['approved'] = True
    run['approver'] = approver
    self.runs.save(run)
    return True