Ejemplo n.º 1
0
 def initializeLibraries(self, browser_config):
     if ((browser_config['remote'] == True) and (browser_config['host'] <> '')):
         from ffprocess_winmo import WinmoProcess
         self._ffprocess = WinmoProcess(browser_config['host'], 
                                        browser_config['port'], 
                                        browser_config['deviceroot'])
         self._ffsetup = FFSetup(self._ffprocess)
         self._ffsetup.initializeRemoteDevice(browser_config)
Ejemplo n.º 2
0
    def __init__(self, remote=False):
        cmanager, platformtype, ffprocess = self.getPlatformType(remote)
        self.CounterManager = cmanager
        self.platform_type = platformtype
        self._ffprocess = ffprocess
        self._hostproc = ffprocess
        self.remote = remote

        self._ffsetup = FFSetup(self._ffprocess)
Ejemplo n.º 3
0
    def initializeLibraries(self, browser_config):
        if browser_config['remote'] == True:
            cmanager, platform_type, ffprocess = self.getPlatformType(False)

            from ffprocess_remote import RemoteProcess
            self._ffprocess = RemoteProcess(browser_config['host'],
                                            browser_config['port'],
                                            browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config, ffprocess)
            self._hostproc = ffprocess
Ejemplo n.º 4
0
Archivo: ttest.py Proyecto: wlach/talos
    def __init__(self, remote = False):
        cmanager, platformtype, ffprocess = self.getPlatformType(remote)
        self.cmanager = cmanager
        self.platform_type = platformtype
        self._ffprocess = ffprocess
        self._hostproc = ffprocess
        self.remote = remote

        self._ffsetup = FFSetup(self._ffprocess)
Ejemplo n.º 5
0
Archivo: ttest.py Proyecto: wlach/talos
    def initializeLibraries(self, browser_config):
        if browser_config['remote'] == True:
            cmanager, platform_type, ffprocess = self.getPlatformType(False)

            from ffprocess_remote import RemoteProcess
            self._ffprocess = RemoteProcess(browser_config['host'], 
                                           browser_config['port'], 
                                           browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config, ffprocess)
            self._hostproc = ffprocess
Ejemplo n.º 6
0
    def __init__(self, remote = False):
        self.remote = remote
        self.cmanager = None
        if self.remote == True:
            self.platform_type = 'win_'
        elif platform.system() == "Linux":
            self.cmanager = __import__('cmanager_linux')
            self.platform_type = 'linux_'
            self._ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version(): #winxp
              self.platform_type = 'win_'
            elif '6.1' in platform.version(): #w7
              self.platform_type = 'w7_'
            else:
              raise talosError('unsupported windows version')
            self.cmanager = __import__('cmanager_win32')
            self._ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            self.cmanager = __import__('cmanager_mac')
            self.platform_type = 'mac_'
            self._ffprocess = MacProcess()

        self._ffsetup = FFSetup(self._ffprocess)
Ejemplo n.º 7
0
class TTest(object):

    _ffsetup = None
    _ffprocess = None
    platform_type = ''

    def __init__(self, remote=False):
        cmanager, platformtype, ffprocess = self.getPlatformType(remote)
        self.CounterManager = cmanager
        self.platform_type = platformtype
        self._ffprocess = ffprocess
        self._hostproc = ffprocess
        self.remote = remote

        self._ffsetup = FFSetup(self._ffprocess)

    def getPlatformType(self, remote):

        _ffprocess = None
        if remote == True:
            platform_type = 'remote_'
            import cmanager
            CounterManager = cmanager.CounterManager
        elif platform.system() == "Linux":
            import cmanager_linux
            CounterManager = cmanager_linux.LinuxCounterManager
            platform_type = 'linux_'
            _ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version():  #winxp
                platform_type = 'win_'
            elif '6.1' in platform.version():  #w7
                platform_type = 'w7_'
            elif '6.2' in platform.version():  #w8
                platform_type = 'w8_'
            else:
                raise talosError('unsupported windows version')
            import cmanager_win32
            CounterManager = cmanager_win32.WinCounterManager
            _ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            import cmanager_mac
            CounterManager = cmanager_mac.MacCounterManager
            platform_type = 'mac_'
            _ffprocess = MacProcess()
        return CounterManager, platform_type, _ffprocess

    def initializeLibraries(self, browser_config):
        if browser_config['remote'] == True:
            cmanager, platform_type, ffprocess = self.getPlatformType(False)

            from ffprocess_remote import RemoteProcess
            self._ffprocess = RemoteProcess(browser_config['host'],
                                            browser_config['port'],
                                            browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config, ffprocess)
            self._hostproc = ffprocess

    def createProfile(self, profile_path, preferences, extensions, webserver):
        # Create the new profile
        temp_dir, profile_dir = self._ffsetup.CreateTempProfileDir(
            profile_path, preferences, extensions, webserver)
        utils.debug("created profile")
        return profile_dir, temp_dir

    def initializeProfile(self, profile_dir, browser_config):
        if not self._ffsetup.InitializeNewProfile(profile_dir, browser_config):
            raise talosError("failed to initialize browser")
        processes = self._ffprocess.checkAllProcesses(
            browser_config['process'], browser_config['child_process'])
        if processes:
            raise talosError("browser failed to close after being initialized")

    def cleanupProfile(self, dir):
        # Delete the temp profile directory  Make it writeable first,
        # because every once in a while browser seems to drop a read-only
        # file into it.
        self._hostproc.removeDirectory(dir)

    def cleanupAndCheckForCrashes(self, browser_config, profile_dir,
                                  test_name):
        """cleanup browser processes and process crashes if found"""

        # cleanup processes
        cleanup_result = self._ffprocess.cleanupProcesses(
            browser_config['process'], browser_config['child_process'],
            browser_config['browser_wait'])

        # find stackwalk binary
        if platform.system() in ('Windows', 'Microsoft'):
            stackwalkpaths = ['win32', 'minidump_stackwalk.exe']
        elif platform.system() == 'Linux':
            if '64' in platform.architecture()[0]:  #are we 64 bit?
                stackwalkpaths = ['linux64', 'minidump_stackwalk']
            else:
                stackwalkpaths = ['linux', 'minidump_stackwalk']
        elif platform.system() == 'Darwin':
            stackwalkpaths = ['osx', 'minidump_stackwalk']
        else:
            # no minidump_stackwalk available for your platform
            return
        stackwalkbin = os.path.join(os.path.dirname(__file__), 'breakpad',
                                    *stackwalkpaths)
        assert os.path.exists(
            stackwalkbin
        ), "minidump_stackwalk binary not found: %s" % stackwalkbin

        # look for minidumps
        minidumpdir = os.path.join(profile_dir, 'minidumps')
        if browser_config['remote'] == True:
            minidumpdir = tempfile.mkdtemp()
            try:
                remoteminidumpdir = profile_dir + '/minidumps/'
                if self._ffprocess.testAgent.dirExists(remoteminidumpdir):
                    self._ffprocess.testAgent.getDirectory(
                        remoteminidumpdir, minidumpdir)
            except mozdevice.DMError:
                print "Remote Device Error: Error getting crash minidumps from device"
                raise

        found = mozcrash.check_for_crashes(minidumpdir,
                                           browser_config['symbols_path'],
                                           stackwalk_binary=stackwalkbin,
                                           test_name=test_name)

        if browser_config['remote'] == True:
            # cleanup dumps on remote
            self._ffprocess.testAgent.removeDir(remoteminidumpdir)
            self._hostproc.removeDirectory(minidumpdir)

        if found:
            raise talosCrash("Found crashes after test run, terminating test")

    def setupRobocopTests(self, browser_config, profile_dir):
        try:
            deviceRoot = self._ffprocess.testAgent.getDeviceRoot()
            fHandle = open("robotium.config", "w")
            fHandle.write("profile=%s\n" % profile_dir)

            remoteLog = deviceRoot + "/" + browser_config['browser_log']
            fHandle.write("logfile=%s\n" % remoteLog)
            fHandle.write("host=http://%s\n" % browser_config['webserver'])
            fHandle.write("rawhost=http://%s\n" % browser_config['webserver'])
            envstr = ""
            delim = ""
            # This is not foolproof and the ideal solution would be to have one env/line instead of a single string
            for key, value in browser_config.get('env', {}).items():
                try:
                    value.index(',')
                    print "Error: Found an ',' in our value, unable to process value."
                except ValueError, e:
                    envstr += "%s%s=%s" % (delim, key, value)
                    delim = ","

            fHandle.write("envvars=%s\n" % envstr)
            fHandle.close()

            self._ffprocess.testAgent.removeFile(
                os.path.join(deviceRoot, "fennec_ids.txt"))
            self._ffprocess.testAgent.removeFile(
                os.path.join(deviceRoot, "robotium.config"))
            self._ffprocess.testAgent.removeFile(remoteLog)
            self._ffprocess.testAgent.pushFile(
                "robotium.config", os.path.join(deviceRoot, "robotium.config"))
            self._ffprocess.testAgent.pushFile(
                browser_config['fennecIDs'],
                os.path.join(deviceRoot, "fennec_ids.txt"))
        except mozdevice.DMError:
            print "Remote Device Error: Error copying files for robocop setup"
            raise
Ejemplo n.º 8
0
Archivo: ttest.py Proyecto: wlach/talos
class TTest(object):

    _ffsetup = None
    _ffprocess = None
    platform_type = ''

    # Regular expression for getting results from most tests
    RESULTS_REGEX = re.compile('__start_report(.*?)__end_report.*?__startTimestamp(.*?)__endTimestamp.*?__startBeforeLaunchTimestamp(.*?)__endBeforeLaunchTimestamp.*?__startAfterTerminationTimestamp(.*?)__endAfterTerminationTimestamp',
                      re.DOTALL | re.MULTILINE)
    # Regular expression to get stats for page load test (Tp) - 
    #should go away once data passing is standardized
    RESULTS_TP_REGEX = re.compile('__start_tp_report(.*?)__end_tp_report.*?__startTimestamp(.*?)__endTimestamp.*?__startBeforeLaunchTimestamp(.*?)__endBeforeLaunchTimestamp.*?__startAfterTerminationTimestamp(.*?)__endAfterTerminationTimestamp',
                      re.DOTALL | re.MULTILINE)
    RESULTS_REGEX_FAIL = re.compile('__FAIL(.*?)__FAIL', re.DOTALL|re.MULTILINE)
    RESULTS_RESPONSIVENESS_REGEX = re.compile('MOZ_EVENT_TRACE\ssample\s\d*?\s(\d*?)$', re.DOTALL|re.MULTILINE)

    def __init__(self, remote = False):
        cmanager, platformtype, ffprocess = self.getPlatformType(remote)
        self.cmanager = cmanager
        self.platform_type = platformtype
        self._ffprocess = ffprocess
        self._hostproc = ffprocess
        self.remote = remote

        self._ffsetup = FFSetup(self._ffprocess)

    def getPlatformType(self, remote):
        cmanager = None
        _ffprocess = None
        if remote == True:
            platform_type = 'win_'
        elif platform.system() == "Linux":
            cmanager = __import__('cmanager_linux')
            platform_type = 'linux_'
            _ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version(): #winxp
                platform_type = 'win_'
            elif '6.1' in platform.version(): #w7
                platform_type = 'w7_'
            else:
                raise talosError('unsupported windows version')
            cmanager = __import__('cmanager_win32')
            _ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            cmanager = __import__('cmanager_mac')
            platform_type = 'mac_'
            _ffprocess = MacProcess()
        return cmanager, platform_type, _ffprocess

    def initializeLibraries(self, browser_config):
        if browser_config['remote'] == True:
            cmanager, platform_type, ffprocess = self.getPlatformType(False)

            from ffprocess_remote import RemoteProcess
            self._ffprocess = RemoteProcess(browser_config['host'], 
                                           browser_config['port'], 
                                           browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config, ffprocess)
            self._hostproc = ffprocess

    def createProfile(self, profile_path, browser_config):
        # Create the new profile
        temp_dir, profile_dir = self._ffsetup.CreateTempProfileDir(profile_path,
                                                     browser_config['preferences'],
                                                     browser_config['extensions'])
        utils.debug("created profile") 
        return profile_dir, temp_dir

    def initializeProfile(self, profile_dir, browser_config):
        if not self._ffsetup.InitializeNewProfile(profile_dir, browser_config):
            raise talosError("failed to initialize browser")
        time.sleep(browser_config['browser_wait'])
        if self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
            raise talosError("browser failed to close after being initialized") 

    def cleanupProfile(self, dir):
        # Delete the temp profile directory  Make it writeable first,
        # because every once in a while browser seems to drop a read-only
        # file into it.
        self._hostproc.removeDirectory(dir)

    def cleanupAndCheckForCrashes(self, browser_config, profile_dir):
        cleanup_result = self._ffprocess.cleanupProcesses(browser_config['process'], 
                                                          browser_config['child_process'], 
                                                          browser_config['browser_wait']) 
        if platform.system() in ('Windows', 'Microsoft'):
            stackwalkpaths = ['win32', 'minidump_stackwalk.exe']
        elif platform.system() == 'Linux':
            if platform.machine() == 'armv6l':
                stackwalkpaths = ['maemo', 'minidump_stackwalk']
            elif '64' in platform.architecture()[0]: #are we 64 bit?
                stackwalkpaths = ['linux64', 'minidump_stackwalk']
            else:
                stackwalkpaths = ['linux', 'minidump_stackwalk']
        elif platform.system() == 'Darwin':
            stackwalkpaths = ['osx', 'minidump_stackwalk']
        else:
            return
        stackwalkbin = os.path.join(os.path.dirname(__file__), 'breakpad', *stackwalkpaths)

        found = False
        minidumpdir = os.path.join(profile_dir, 'minidumps')
        if browser_config['remote'] == True:
            minidumpdir = tempfile.mkdtemp()
            self._ffprocess.testAgent.getDirectory(profile_dir + '/minidumps/', minidumpdir)
        
        for dump in glob.glob(os.path.join(minidumpdir, '*.dmp')):
            utils.noisy("Found crashdump: " + dump)
            if browser_config['symbols_path']:
                nullfd = open(os.devnull, 'w')
                subprocess.call([stackwalkbin, dump, browser_config['symbols_path']], stderr=nullfd)
                nullfd.close()
            os.remove(dump)
            found = True

        if browser_config['remote'] == True:
            self._hostproc.removeDirectory(minidumpdir)
   
        if found:
            if cleanup_result:
                raise talosError("stack found after process termination (" + cleanup_result+ ")")
            else:
                raise talosError("crash during run (stack found)")


    def runTest(self, browser_config, test_config):
        """
            Runs an url based test on the browser as specified in the browser_config dictionary
  
        Args:
            browser_config:  Dictionary of configuration options for the browser (paths, prefs, etc)
            test_config   :  Dictionary of configuration for the given test (url, cycles, counters, etc)

        """
        self.initializeLibraries(browser_config)

        utils.debug("operating with platform_type : " + self.platform_type)
        counters = test_config[self.platform_type + 'counters']
        resolution = test_config['resolution']
        all_browser_results = []
        all_counter_results = []
        format = ""
        utils.setEnvironmentVars(browser_config['env'])
        utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})

        if browser_config['symbols_path']:
            utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
        else:
            utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})

        utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})

        profile_dir = None

        try:
            running_processes = self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process'])
            if running_processes:
                msg = " already running before testing started (unclean system)"
                utils.debug(browser_config['process'] + msg)
                raise talosError("Found processes still running: %s. Please close them before running talos." % ", ".join(running_processes))

            for bundlename in browser_config['bundles']:
                self._ffsetup.InstallBundleInBrowser(browser_config['browser_path'],
                                                     bundlename,
                                                     browser_config['bundles'][bundlename])
  
            # add any provided directories to the installed browser
            for dir in browser_config['dirs']:
                self._ffsetup.InstallInBrowser(browser_config['browser_path'], 
                                            browser_config['dirs'][dir])
  
            # make profile path work cross-platform
            test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
            profile_dir, temp_dir = self.createProfile(test_config['profile_path'], browser_config)
            if os.path.isfile(browser_config['browser_log']):
                os.chmod(browser_config['browser_log'], 0777)
                os.remove(browser_config['browser_log'])
            self.initializeProfile(profile_dir, browser_config)
    
            utils.debug("initialized " + browser_config['process'])
            if test_config['shutdown']:
                shutdown = []
            if 'responsiveness' in test_config and test_config['responsiveness']:
               utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP': '1'})
               utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD': '20'})
               utils.setEnvironmentVars({'MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL': '10'})
               responsiveness = []

            for i in range(test_config['cycles']):
                if os.path.isfile(browser_config['browser_log']):
                    os.chmod(browser_config['browser_log'], 0777)
                    os.remove(browser_config['browser_log'])
                time.sleep(browser_config['browser_wait']) #wait out the browser closing
                # check to see if the previous cycle is still hanging around 
                if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
                    raise talosError("previous cycle still running")

                # Execute the test's head script if there is one
                if 'head' in test_config:
                    try:
                        subprocess.call(['python', test_config['head']])
                    except:
                        raise talosError("error executing head script: %s" % sys.exc_info()[0])

                # Run the test 
                browser_results = ""
                if 'timeout' in test_config:
                     timeout = test_config['timeout']
                else:
                    timeout = 7200 # 2 hours
                total_time = 0
                output = ''
                url = test_config['url']
                command_line = self._ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'],
                                                                        browser_config['extra_args'],
                                                                        profile_dir, 
                                                                        url)
  
                utils.debug("command line: " + command_line)

                b_log = browser_config['browser_log']
                if (self.remote == True):
                    b_log = browser_config['deviceroot'] + '/' + browser_config['browser_log']

                b_cmd = self._ffprocess.GenerateBControllerCommandLine(command_line, browser_config, test_config)
                process = subprocess.Popen(b_cmd, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
  
                #give browser a chance to open
                # this could mean that we are losing the first couple of data points
                #   as the tests starts, but if we don't provide
                # some time for the browser to start we have trouble connecting the CounterManager to it
                time.sleep(browser_config['browser_wait'])
                #set up the counters for this test
                if counters:
                    cm = self.cmanager.CounterManager(self._ffprocess, browser_config['process'], counters)
                counter_results = {}
                for counter in counters:
                    counter_results[counter] = []

                startTime = -1
                dumpResult = ""
                #the main test loop, monitors counters and checks for browser output
                while total_time < timeout:
                    # Sleep for [resolution] seconds
                    time.sleep(resolution)
                    total_time += resolution
                    fileData = self._ffprocess.getFile(b_log)
                    if (len(fileData) > 0):
                        utils.noisy(fileData.replace(dumpResult, ''))
                        dumpResult = fileData
        

                    # Get the output from all the possible counters
                    for count_type in counters:
                        val = cm.getCounterValue(count_type)
                        if (val):
                            counter_results[count_type].append(val)
                    if process.poll() != None: #browser_controller completed, file now full
                        timed_out = False
                        break
                        
                if total_time >= timeout:
                    raise talosError("timeout exceeded")
                else:
                    #stop the counter manager since this test is complete
                    if counters:
                        cm.stopMonitor()

                    if not os.path.isfile(browser_config['browser_log']):
                        raise talosError("no output from browser")
                    results_file = open(browser_config['browser_log'], "r")
                    results_raw = results_file.read()
                    results_file.close()
  
                    match = self.RESULTS_REGEX.search(results_raw)
                    tpmatch = self.RESULTS_TP_REGEX.search(results_raw)
                    failmatch = self.RESULTS_REGEX_FAIL.search(results_raw)
                    if match:
                        browser_results += match.group(1)
                        startTime = int(match.group(2))
                        endTime = int(match.group(4))
                        format = "tsformat"
                    #TODO: this a stop gap until all of the tests start outputting the same format
                    elif tpmatch:
                        match = tpmatch
                        browser_results += match.group(1)
                        startTime = int(match.group(2))
                        endTime = int(match.group(4))
                        format = "tpformat"
                    elif failmatch:
                        match = failmatch
                        browser_results += match.group(1)
                        raise talosError(match.group(1))
                    else:
                        raise talosError("unrecognized output format")
  
                time.sleep(browser_config['browser_wait']) 
                #clean up any stray browser processes
                self.cleanupAndCheckForCrashes(browser_config, profile_dir)
                #clean up the bcontroller process
                timer = 0
                while ((process.poll() is None) and timer < browser_config['browser_wait']):
                    time.sleep(1)
                    timer+=1
 
                if test_config['shutdown']:
                    shutdown.append(endTime - startTime)
                if 'responsiveness' in test_config and test_config['responsiveness']:
                    responsiveness = self.RESULTS_RESPONSIVENESS_REGEX.findall(results_raw)

                all_browser_results.append(browser_results)
                all_counter_results.append(counter_results)

                # Execute the test's tail script if there is one
                if 'tail' in test_config:
                    try:
                        subprocess.call(['python', test_config['tail']])
                    except:
                        raise talosError("error executing tail script: %s" % sys.exc_info()[0])

            self.cleanupProfile(temp_dir)
            utils.restoreEnvironmentVars()
            if test_config['shutdown']:
                all_counter_results.append({'shutdown' : shutdown})      
            if 'responsiveness' in test_config and test_config['responsiveness']:
                all_counter_results.append({'responsiveness' : responsiveness})      
            return (all_browser_results, all_counter_results, format)
        except:
            try:
                if 'cm' in vars():
                    cm.stopMonitor()

                if os.path.isfile(browser_config['browser_log']):
                    results_file = open(browser_config['browser_log'], "r")
                    results_raw = results_file.read()
                    results_file.close()
                    utils.noisy(results_raw)

                if profile_dir:
                    try:
                        self.cleanupAndCheckForCrashes(browser_config, profile_dir)
                    except talosError:
                        pass

                if vars().has_key('temp_dir'):
                    self.cleanupProfile(temp_dir)
            except talosError, te:
                utils.debug("cleanup error: " + te.msg)
            except:
                utils.debug("unknown error during cleanup")
            raise
Ejemplo n.º 9
0
class TTest(object):

    _ffsetup = None
    _ffprocess = None
    platform_type = ''

    def __init__(self, remote = False):
        cmanager, platformtype, ffprocess = self.getPlatformType(remote)
        self.CounterManager = cmanager
        self.platform_type = platformtype
        self._ffprocess = ffprocess
        self._hostproc = ffprocess
        self.remote = remote

        self._ffsetup = FFSetup(self._ffprocess)

    def getPlatformType(self, remote):

        _ffprocess = None
        if remote == True:
            platform_type = 'remote_'
            import cmanager
            CounterManager = cmanager.CounterManager
        elif platform.system() == "Linux":
            import cmanager_linux
            CounterManager = cmanager_linux.LinuxCounterManager
            platform_type = 'linux_'
            _ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version(): #winxp
                platform_type = 'win_'
            elif '6.1' in platform.version(): #w7
                platform_type = 'w7_'
            elif '6.2' in platform.version(): #w8
                platform_type = 'w8_'
            else:
                raise talosError('unsupported windows version')
            import cmanager_win32
            CounterManager = cmanager_win32.WinCounterManager
            _ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            import cmanager_mac
            CounterManager = cmanager_mac.MacCounterManager
            platform_type = 'mac_'
            _ffprocess = MacProcess()
        return CounterManager, platform_type, _ffprocess

    def initializeLibraries(self, browser_config):
        if browser_config['remote'] == True:
            cmanager, platform_type, ffprocess = self.getPlatformType(False)

            from ffprocess_remote import RemoteProcess
            self._ffprocess = RemoteProcess(browser_config['host'],
                                            browser_config['port'],
                                            browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config, ffprocess)
            self._hostproc = ffprocess

    def createProfile(self, profile_path, preferences, extensions, webserver):
        # Create the new profile
        temp_dir, profile_dir = self._ffsetup.CreateTempProfileDir(profile_path,
                                                     preferences,
                                                     extensions,
                                                     webserver)
        utils.debug("created profile")
        return profile_dir, temp_dir

    def initializeProfile(self, profile_dir, browser_config):
        if not self._ffsetup.InitializeNewProfile(profile_dir, browser_config):
            raise talosError("failed to initialize browser")
        processes = self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process'])
        if processes:
            raise talosError("browser failed to close after being initialized")

    def cleanupProfile(self, dir):
        # Delete the temp profile directory  Make it writeable first,
        # because every once in a while browser seems to drop a read-only
        # file into it.
        self._hostproc.removeDirectory(dir)

    def cleanupAndCheckForCrashes(self, browser_config, profile_dir, test_name):
        """cleanup browser processes and process crashes if found"""

        # cleanup processes
        cleanup_result = self._ffprocess.cleanupProcesses(browser_config['process'],
                                                          browser_config['child_process'],
                                                          browser_config['browser_wait'])

        # find stackwalk binary
        if platform.system() in ('Windows', 'Microsoft'):
            stackwalkpaths = ['win32', 'minidump_stackwalk.exe']
        elif platform.system() == 'Linux':
            if '64' in platform.architecture()[0]: #are we 64 bit?
                stackwalkpaths = ['linux64', 'minidump_stackwalk']
            else:
                stackwalkpaths = ['linux', 'minidump_stackwalk']
        elif platform.system() == 'Darwin':
            stackwalkpaths = ['osx', 'minidump_stackwalk']
        else:
            # no minidump_stackwalk available for your platform
            return
        stackwalkbin = os.path.join(os.path.dirname(__file__), 'breakpad', *stackwalkpaths)
        assert os.path.exists(stackwalkbin), "minidump_stackwalk binary not found: %s" % stackwalkbin

        # look for minidumps
        minidumpdir = os.path.join(profile_dir, 'minidumps')
        if browser_config['remote'] == True:
            minidumpdir = tempfile.mkdtemp()
            try:
                remoteminidumpdir = profile_dir + '/minidumps/'
                if self._ffprocess.testAgent.dirExists(remoteminidumpdir):
                    self._ffprocess.testAgent.getDirectory(remoteminidumpdir, minidumpdir)
            except mozdevice.DMError:
                print "Remote Device Error: Error getting crash minidumps from device"
                raise

        found = mozcrash.check_for_crashes(minidumpdir, 
                                           browser_config['symbols_path'], 
                                           stackwalk_binary=stackwalkbin,
                                           test_name=test_name)

        if browser_config['remote'] == True:
            # cleanup dumps on remote
            self._ffprocess.testAgent.removeDir(remoteminidumpdir)
            self._hostproc.removeDirectory(minidumpdir)

        if found:
            raise talosCrash("Found crashes after test run, terminating test")

    def setupRobocopTests(self, browser_config, profile_dir):
        try:
            deviceRoot = self._ffprocess.testAgent.getDeviceRoot()
            fHandle = open("robotium.config", "w")
            fHandle.write("profile=%s\n" % profile_dir)

            remoteLog = deviceRoot + "/" + browser_config['browser_log']
            fHandle.write("logfile=%s\n" % remoteLog)
            fHandle.write("host=http://%s\n" % browser_config['webserver'])
            fHandle.write("rawhost=http://%s\n" % browser_config['webserver'])
            envstr = ""
            delim = ""
            # This is not foolproof and the ideal solution would be to have one env/line instead of a single string
            for key, value in browser_config.get('env', {}).items():
                try:
                    value.index(',')
                    print "Error: Found an ',' in our value, unable to process value."
                except ValueError, e:
                    envstr += "%s%s=%s" % (delim, key, value)
                    delim = ","

            fHandle.write("envvars=%s\n" % envstr)
            fHandle.close()

            self._ffprocess.testAgent.removeFile(os.path.join(deviceRoot, "fennec_ids.txt"))
            self._ffprocess.testAgent.removeFile(os.path.join(deviceRoot, "robotium.config"))
            self._ffprocess.testAgent.removeFile(remoteLog)
            self._ffprocess.testAgent.pushFile("robotium.config", os.path.join(deviceRoot, "robotium.config"))
            self._ffprocess.testAgent.pushFile(browser_config['fennecIDs'], os.path.join(deviceRoot, "fennec_ids.txt"))
        except mozdevice.DMError:
            print "Remote Device Error: Error copying files for robocop setup"
            raise
Ejemplo n.º 10
0
class TTest(object):

    _ffsetup = None
    _ffprocess = None
    platform_type = ''

    # Regular expression for getting results from most tests
    RESULTS_REGEX = re.compile('__start_report(.*?)__end_report.*?__startTimestamp(.*?)__endTimestamp.*?__startSecondTimestamp(.*?)__endSecondTimestamp',
                      re.DOTALL | re.MULTILINE)
    # Regular expression to get stats for page load test (Tp) - 
    #should go away once data passing is standardized
    RESULTS_TP_REGEX = re.compile('__start_tp_report(.*?)__end_tp_report.*?__startTimestamp(.*?)__endTimestamp.*?__startSecondTimestamp(.*?)__endSecondTimestamp',
                      re.DOTALL | re.MULTILINE)
    RESULTS_REGEX_FAIL = re.compile('__FAIL(.*?)__FAIL', re.DOTALL|re.MULTILINE)

    def __init__(self, remote = False):
        self.remote = remote
        self.cmanager = None
        if self.remote == True:
            self.platform_type = 'win_'
        elif platform.system() == "Linux":
            self.cmanager = __import__('cmanager_linux')
            self.platform_type = 'linux_'
            self._ffprocess = LinuxProcess()
        elif platform.system() in ("Windows", "Microsoft"):
            if '5.1' in platform.version(): #winxp
              self.platform_type = 'win_'
            elif '6.1' in platform.version(): #w7
              self.platform_type = 'w7_'
            else:
              raise talosError('unsupported windows version')
            self.cmanager = __import__('cmanager_win32')
            self._ffprocess = Win32Process()
        elif platform.system() == "Darwin":
            self.cmanager = __import__('cmanager_mac')
            self.platform_type = 'mac_'
            self._ffprocess = MacProcess()

        self._ffsetup = FFSetup(self._ffprocess)

    def initializeLibraries(self, browser_config):
        if ((browser_config['remote'] == True) and (browser_config['host'] <> '')):
            from ffprocess_winmo import WinmoProcess
            self._ffprocess = WinmoProcess(browser_config['host'], 
                                           browser_config['port'], 
                                           browser_config['deviceroot'])
            self._ffsetup = FFSetup(self._ffprocess)
            self._ffsetup.initializeRemoteDevice(browser_config)

    def createProfile(self, profile_path, browser_config):
        # Create the new profile
        temp_dir, profile_dir = self._ffsetup.CreateTempProfileDir(profile_path,
                                                     browser_config['preferences'],
                                                     browser_config['extensions'])
        utils.debug("created profile") 
        return profile_dir, temp_dir

    def initializeProfile(self, profile_dir, browser_config):
        if not (self._ffsetup.InitializeNewProfile(browser_config['browser_path'], 
                                                    browser_config['process'],
                                                    browser_config['child_process'],
                                                    browser_config['browser_wait'],
                                                    browser_config['extra_args'], 
                                                    profile_dir, 
                                                    browser_config['init_url'],
                                                    browser_config['browser_log'])):
            raise talosError("failed to initialize browser")
        time.sleep(browser_config['browser_wait'])
        if self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
            raise talosError("browser failed to close after being initialized") 

    def cleanupProfile(self, dir):
        # Delete the temp profile directory  Make it writeable first,
        # because every once in a while browser seems to drop a read-only
        # file into it.
        self._ffprocess.removeDirectory(dir)

    def checkForCrashes(self, browser_config, profile_dir):
        if platform.system() in ('Windows', 'Microsoft'):
            stackwalkpaths = ['win32', 'minidump_stackwalk.exe']
        elif platform.system() == 'Linux':
            if platform.machine() == 'armv6l':
                stackwalkpaths = ['maemo', 'minidump_stackwalk']
            elif '64' in platform.architecture()[0]: #are we 64 bit?
                stackwalkpaths = ['linux64', 'minidump_stackwalk']
            else:
                stackwalkpaths = ['linux', 'minidump_stackwalk']
        elif platform.system() == 'Darwin':
            stackwalkpaths = ['osx', 'minidump_stackwalk']
        else:
            return
        stackwalkbin = os.path.join(os.path.dirname(__file__), 'breakpad', *stackwalkpaths)

        found = False
        for dump in glob.glob(os.path.join(profile_dir, 'minidumps', '*.dmp')):
            utils.noisy("Found crashdump: " + dump)
            if browser_config['symbols_path']:
                nullfd = open(os.devnull, 'w')
                subprocess.call([stackwalkbin, dump, browser_config['symbols_path']], stderr=nullfd)
                nullfd.close()
            os.remove(dump)
            found = True

        if found:
            raise talosError("crash during run (stack found)")

    def runTest(self, browser_config, test_config):
        """
            Runs an url based test on the browser as specified in the browser_config dictionary
  
        Args:
            browser_config:  Dictionary of configuration options for the browser (paths, prefs, etc)
            test_config   :  Dictionary of configuration for the given test (url, cycles, counters, etc)

        """
        self.initializeLibraries(browser_config)
 
        utils.debug("operating with platform_type : " + self.platform_type)
        counters = test_config[self.platform_type + 'counters']
        resolution = test_config['resolution']
        all_browser_results = []
        all_counter_results = []
        format = ""
        utils.setEnvironmentVars(browser_config['env'])
        utils.setEnvironmentVars({'MOZ_CRASHREPORTER_NO_REPORT': '1'})

        if browser_config['symbols_path']:
            utils.setEnvironmentVars({'MOZ_CRASHREPORTER': '1'})
        else:
            utils.setEnvironmentVars({'MOZ_CRASHREPORTER_DISABLE': '1'})

        utils.setEnvironmentVars({"LD_LIBRARY_PATH" : os.path.dirname(browser_config['browser_path'])})

        profile_dir = None

        try:
            if self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
                msg = " already running before testing started (unclean system)"
                utils.debug(browser_config['process'] + msg)
                raise talosError("system not clean")

            for bundlename in browser_config['bundles']:
                self._ffsetup.InstallBundleInBrowser(browser_config['browser_path'],
                                                     bundlename,
                                                     browser_config['bundles'][bundlename])
  
            # add any provided directories to the installed browser
            for dir in browser_config['dirs']:
                self._ffsetup.InstallInBrowser(browser_config['browser_path'], 
                                            browser_config['dirs'][dir])
  
            # make profile path work cross-platform
            test_config['profile_path'] = os.path.normpath(test_config['profile_path'])
            profile_dir, temp_dir = self.createProfile(test_config['profile_path'], browser_config)
            if os.path.isfile(browser_config['browser_log']):
                os.chmod(browser_config['browser_log'], 0777)
                os.remove(browser_config['browser_log'])
            self.initializeProfile(profile_dir, browser_config)
    
            utils.debug("initialized " + browser_config['process'])
            if test_config['shutdown']:
                shutdown = []

            for i in range(test_config['cycles']):
                if os.path.isfile(browser_config['browser_log']):
                    os.chmod(browser_config['browser_log'], 0777)
                    os.remove(browser_config['browser_log'])
                time.sleep(browser_config['browser_wait']) #wait out the browser closing
                # check to see if the previous cycle is still hanging around 
                if (i > 0) and self._ffprocess.checkAllProcesses(browser_config['process'], browser_config['child_process']):
                    raise talosError("previous cycle still running")

                # Execute the test's head script if there is one
                if 'head' in test_config:
                    try:
                        subprocess.call(['python', test_config['head']])
                    except:
                        raise talosError("error executing head script: %s" % sys.exc_info()[0])

                # Run the test 
                browser_results = ""
                if 'timeout' in test_config:
                     timeout = test_config['timeout']
                else:
                    timeout = 7200 # 2 hours
                total_time = 0
                output = ''
                url = test_config['url']
                command_line = self._ffprocess.GenerateBrowserCommandLine(browser_config['browser_path'],
                                                                        browser_config['extra_args'],
                                                                        profile_dir, 
                                                                        url)
  
                utils.debug("command line: " + command_line)
 
                b_cmd = 'python bcontroller.py --command "%s"' % (command_line)
                b_cmd += ' --child_process %s ' % (browser_config['child_process'])
                b_cmd += ' --name %s ' % (browser_config['process'])
                b_cmd += ' --timeout %d ' % (browser_config['browser_wait'])
                b_cmd += ' --log %s ' % (browser_config['browser_log'])
                if 'url_mod' in test_config:
                    b_cmd += ' --mod "%s" ' % (test_config['url_mod'])

                if (self.remote == True):
                    b_cmd += ' --host "%s" ' % (browser_config['host'])
                    b_cmd += ' --port "%s" ' % (browser_config['port'])
                    b_cmd += ' --deviceRoot "%s" ' % (browser_config['deviceroot'])

                process = subprocess.Popen(b_cmd, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
  
                #give browser a chance to open
                # this could mean that we are losing the first couple of data points
                #   as the tests starts, but if we don't provide
                # some time for the browser to start we have trouble connecting the CounterManager to it
                time.sleep(browser_config['browser_wait'])
                #set up the counters for this test
                if counters:
                    cm = self.cmanager.CounterManager(self._ffprocess, browser_config['process'], counters)
                counter_results = {}
                for counter in counters:
                    counter_results[counter] = []

                startTime = -1
                dumpResult = ""
                #the main test loop, monitors counters and checks for browser ouptut
                while total_time < timeout:
                    # Sleep for [resolution] seconds
                    time.sleep(resolution)
                    total_time += resolution
                    fileData = self._ffprocess.getFile(browser_config['browser_log'])
                    if (len(fileData) > 0):
                        utils.noisy(fileData.replace(dumpResult, ''))
                        dumpResult = fileData
        

                    # Get the output from all the possible counters
                    for count_type in counters:
                        val = cm.getCounterValue(count_type)
                        if (val):
                            counter_results[count_type].append(val)
                    if process.poll() != None: #browser_controller completed, file now full
                        timed_out = False
                        break
                        
                if total_time >= timeout:
                    raise talosError("timeout exceeded")
                else:
                    #stop the counter manager since this test is complete
                    if counters:
                        cm.stopMonitor()

                    if not os.path.isfile(browser_config['browser_log']):
                        raise talosError("no output from browser")
                    results_file = open(browser_config['browser_log'], "r")
                    results_raw = results_file.read()
                    results_file.close()
  
                    match = self.RESULTS_REGEX.search(results_raw)
                    tpmatch = self.RESULTS_TP_REGEX.search(results_raw)
                    failmatch = self.RESULTS_REGEX_FAIL.search(results_raw)
                    if match:
                        browser_results += match.group(1)
                        startTime = int(match.group(2))
                        endTime = int(match.group(3))
                        format = "tsformat"
                    #TODO: this a stop gap until all of the tests start outputting the same format
                    elif tpmatch:
                        match = tpmatch
                        browser_results += match.group(1)
                        startTime = int(match.group(2))
                        endTime = int(match.group(3))
                        format = "tpformat"
                    elif failmatch:
                        match = failmatch
                        browser_results += match.group(1)
                        raise talosError(match.group(1))
                    else:
                        raise talosError("unrecognized output format")
  
                time.sleep(browser_config['browser_wait']) 
                #clean up the process
                timer = 0
                while ((process.poll() is None) and timer < browser_config['browser_wait']):
                    time.sleep(1)
                    timer+=1
 
                if test_config['shutdown']:
                    shutdown.append(endTime - startTime)

                self.checkForCrashes(browser_config, profile_dir)

                all_browser_results.append(browser_results)
                all_counter_results.append(counter_results)

                # Execute the test's tail script if there is one
                if 'tail' in test_config:
                    try:
                        subprocess.call(['python', test_config['tail']])
                    except:
                        raise talosError("error executing tail script: %s" % sys.exc_info()[0])

            self._ffprocess.cleanupProcesses(browser_config['process'], browser_config['child_process'], browser_config['browser_wait']) 
            self.cleanupProfile(temp_dir)

            utils.restoreEnvironmentVars()
            if test_config['shutdown']:
                all_counter_results.append({'shutdown' : shutdown})      
            return (all_browser_results, all_counter_results, format)
        except:
            try:
                if 'cm' in vars():
                    cm.stopMonitor()

                if os.path.isfile(browser_config['browser_log']):
                    results_file = open(browser_config['browser_log'], "r")
                    results_raw = results_file.read()
                    results_file.close()
                    utils.noisy(results_raw)

                self._ffprocess.cleanupProcesses(browser_config['process'], browser_config['child_process'], 
                                                browser_config['browser_wait'])

                if profile_dir:
                    try:
                        self.checkForCrashes(browser_config, profile_dir)
                    except talosError:
                        pass

                if vars().has_key('temp_dir'):
                    self.cleanupProfile(temp_dir)
            except talosError, te:
                utils.debug("cleanup error: " + te.msg)
            except:
                utils.debug("unknown error during cleanup")
            raise