Example #1
0
def runTest(browser_config, test_config):
  """
  Runs an url based test on the browser as specified in the browser_config dictionary
  
  Args:
    browser_config:  Dictionary of configuration options for the browser (paths, prefs, etc)
    test_config   :  Dictionary of configuration for the given test (url, cycles, counters, etc)
  
  """
 
  utils.debug("operating with platform_type : " + platform_type)
  counters = test_config[platform_type + 'counters']
  resolution = test_config['resolution']
  all_browser_results = []
  all_counter_results = []
  format = ""
  utils.setEnvironmentVars(browser_config['env'])
  utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})

  if browser_config['symbols_path']:
      utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
  else:
      utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})

  utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})

  profile_dir = None

  try:
    if ffprocess.checkAllProcesses(browser_config['process']):
      utils.debug(browser_config['process'] + " already running before testing started (unclean system)")
      raise talosError("system not clean")
  
    # add any provided directories to the installed browser
    for dir in browser_config['dirs']:
      ffsetup.InstallInBrowser(browser_config['browser_path'], browser_config['dirs'][dir])
  
    # make profile path work cross-platform
    test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
    profile_dir, temp_dir = test_config['profile_path'], test_config['profile_path']
    if os.path.isfile(browser_config['browser_log']):
      os.chmod(browser_config['browser_log'], 0777)
      os.remove(browser_config['browser_log'])
    #initializeProfile(profile_dir, browser_config)
    
    utils.debug("initialized " + browser_config['process'])
    if test_config['shutdown']:
      shutdown = []
  
    for i in range(test_config['cycles']):
      if os.path.isfile(browser_config['browser_log']):
        os.chmod(browser_config['browser_log'], 0777)
        os.remove(browser_config['browser_log'])
      time.sleep(browser_config['browser_wait']) #wait out the browser closing
      # check to see if the previous cycle is still hanging around 
      if (i > 0) and ffprocess.checkAllProcesses(browser_config['process']):
        raise talosError("previous cycle still running")
      # Run the test 
      browser_results = ""
      if 'timeout' in test_config:
        timeout = test_config['timeout']
      else:
        timeout = 7200 # 2 hours
      total_time = 0
      output = ''
      url = test_config['url']
      command_line = ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'], browser_config['extra_args'], profile_dir, url)
  
      utils.debug("command line: " + command_line)
      bcontroller = path('talos/bcontroller.py') 
      if 'url_mod' in test_config:
        process = subprocess.Popen('python %s --command "%s" --mod "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, test_config['url_mod'], browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
      else:
        process = subprocess.Popen('python %s --command "%s" --name %s --timeout %d --log %s' % (bcontroller, command_line, browser_config['process'], browser_config['browser_wait'], browser_config['browser_log']), universal_newlines=True, shell=True, bufsize=0, env=os.environ)
  
      #give browser a chance to open
      # this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
      # some time for the browser to start we have trouble connecting the CounterManager to it
      time.sleep(browser_config['browser_wait'])
      #set up the counters for this test
      if counters:
        cm = CounterManager(browser_config['process'], counters)
        cm.startMonitor()
      counter_results = {}
      for counter in counters:
        counter_results[counter] = []
     
      startTime = -1
      while total_time < timeout: #the main test loop, monitors counters and checks for browser ouptut
        # Sleep for [resolution] seconds
        time.sleep(resolution)
        total_time += resolution
        
        # Get the output from all the possible counters
        for count_type in counters:
          val = cm.getCounterValue(count_type)
          if (val):
            counter_results[count_type].append(val)
        if process.poll() != None: #browser_controller completed, file now full
          #stop the counter manager since this test is complete
          if counters:
            cm.stopMonitor()
          if not os.path.isfile(browser_config['browser_log']):
            raise talosError("no output from browser")
          results_file = open(browser_config['browser_log'], "r")
          results_raw = results_file.read()
          results_file.close()
          utils.noisy(results_raw)
  
          match = RESULTS_REGEX.search(results_raw)
          if match:
            browser_results += match.group(1)
            startTime = int(match.group(2))
            endTime = int(match.group(3))
            utils.debug("Matched basic results: " + browser_results)
            format = "tsformat"
            break
          #TODO: this a stop gap until all of the tests start outputting the same format
          match = RESULTS_TP_REGEX.search(results_raw)
          if match:
            browser_results += match.group(1)
            startTime = int(match.group(2))
            endTime = int(match.group(3))
            utils.debug("Matched tp results: " + browser_results)
            format = "tpformat"
            break
          match = RESULTS_REGEX_FAIL.search(results_raw)
          if match:
            browser_results += match.group(1)
            utils.debug("Matched fail results: " + browser_results)
            raise talosError(match.group(1))
          raise talosError("unrecognized output format")
  
      if total_time >= timeout:
        raise talosError("timeout exceeded")
  
      time.sleep(browser_config['browser_wait']) 
      #clean up the process
      timer = 0
      while ((process.poll() is None) and timer < browser_config['browser_wait']):
        time.sleep(1)
        timer+=1
 
      if test_config['shutdown']:
          shutdown.append(endTime - startTime)

      checkForCrashes(browser_config, profile_dir)

      utils.debug("Completed test with: " + browser_results)
  
      all_browser_results.append(browser_results)
      all_counter_results.append(counter_results)
     
    ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait']) 
    #cleanupProfile(temp_dir)

    utils.restoreEnvironmentVars()
    if test_config['shutdown']:
      all_counter_results.append({'shutdown' : shutdown})      
    return (all_browser_results, all_counter_results, format)
  except:
    try:
      if 'cm' in vars():
        cm.stopMonitor()

      if os.path.isfile(browser_config['browser_log']):
        results_file = open(browser_config['browser_log'], "r")
        results_raw = results_file.read()
        results_file.close()
        utils.noisy(results_raw)

      ffprocess.cleanupProcesses(browser_config['process'], browser_config['browser_wait'])

      if profile_dir:
          try:
              checkForCrashes(browser_config, profile_dir)
          except talosError:
              pass

      if vars().has_key('temp_dir'):
        pass
        #cleanupProfile(temp_dir)
    except talosError, te:
      utils.debug("cleanup error: " + te.msg)
    except:
      utils.debug("unknown error during cleanup")
    raise
Example #2
0
def initializeProfile(profile_dir, browser_config):
  if not (ffsetup.InitializeNewProfile(browser_config['browser_path'], browser_config['process'], browser_config['browser_wait'], browser_config['extra_args'], profile_dir, browser_config['init_url'], browser_config['browser_log'])):
     raise talosError("failed to initialize browser")
  time.sleep(browser_config['browser_wait'])
  if ffprocess.checkAllProcesses(browser_config['process']):
     raise talosError("browser failed to close after being initialized") 
Example #3
0
def test_file(filename):
  """Runs the talos tests on the given config file and generates a report.
  
  Args:
    filename: the name of the file to run the tests on
  """
  
  browser_config = []
  tests = []
  title = ''
  testdate = ''
  csv_dir = ''
  results_server = ''
  results_link = ''
  results = {}
  
  # Read in the profile info from the YAML config file
  config_file = open(filename, 'r')
  yaml_config = yaml.load(config_file)
  config_file.close()
  for item in yaml_config:
    if item == 'title':
      title = yaml_config[item]
    elif item == 'testdate':
      testdate = yaml_config[item]
    elif item == 'csv_dir':
       csv_dir = os.path.normpath(yaml_config[item])
       if not os.path.exists(csv_dir):
         print "FAIL: path \"" + csv_dir + "\" does not exist"
         sys.exit(0)
    elif item == 'results_server':
       results_server = yaml_config[item]
    elif item == 'results_link' :
       results_link = yaml_config[item]
  if (results_link != results_server != ''):
    if not post_file.link_exists(results_server, results_link):
      sys.exit(0)
  browser_config = {'preferences'  : yaml_config['preferences'],
                    'extensions'   : yaml_config['extensions'],
                    'browser_path' : yaml_config['browser_path'],
                    'browser_log'  : yaml_config['browser_log'],
                    'symbols_path' : yaml_config.get('symbols_path', None),
                    'browser_wait' : yaml_config['browser_wait'],
                    'process'      : yaml_config['process'],
                    'extra_args'   : yaml_config['extra_args'],
                    'branch'       : yaml_config['branch'],
                    'buildid'      : yaml_config['buildid'],
                    'env'          : yaml_config['env'],
                    'dirs'         : yaml_config['dirs'],
                    'init_url'     : yaml_config['init_url']}
  if 'branch_name' in yaml_config:
      browser_config['branch_name'] = yaml_config['branch_name']
  if 'test_name_extension' in yaml_config:
      browser_config['test_name_extension'] = yaml_config['test_name_extension']
  else:
      browser_config['test_name_extension'] = ''
  #normalize paths to work accross platforms
  browser_config['browser_path'] = os.path.normpath(browser_config['browser_path'])
  for dir in browser_config['dirs']:
    browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir])
  tests = yaml_config['tests']
  config_file.close()
  if (testdate != ''):
    date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
  else:
    date = int(time.time()) #TODO get this into own file
  utils.debug("using testdate: %d" % date)
  utils.debug("actual date: %d" % int(time.time()))
  print 'RETURN:s: %s' % title
  #pull buildid & sourcestamp from browser
  browser_config = browserInfo(browser_config)

  if ffprocess.checkAllProcesses(browser_config['process']):
    print "FAIL: all firefox processes must be closed before tests can be run"
    sys.exit(0)

  utils.startTimer()
  utils.stamped_msg(title, "Started")
  for test in tests:
    testname = test['name']
    utils.stamped_msg("Running test " + testname, "Started")
    try:
      browser_dump, counter_dump, print_format = ttest.runTest(browser_config, test)
      utils.debug("Received test results: " + " ".join(browser_dump))
      results[testname] = [browser_dump, counter_dump, print_format]
      # If we're doing CSV, write this test immediately (bug 419367)
      if csv_dir != '':
        send_to_csv(csv_dir, {testname : results[testname]})
    except talosError, e:
      utils.stamped_msg("Failed " + testname, "Stopped")
      print 'FAIL: Busted: ' + testname
      print 'FAIL: ' + e.msg.replace('\n','\nRETURN:')
    utils.stamped_msg("Completed test " + testname, "Stopped")