def update_windows_certificates(self): """ Update the root Windows certificates""" try: cert_file = os.path.join(self.persistent_work_dir, 'root_certs.sst') if not os.path.isdir(self.persistent_work_dir): os.makedirs(self.persistent_work_dir) needs_update = True if os.path.isfile(cert_file): days = (time.time() - os.path.getmtime(cert_file)) / 86400 if days < 5: needs_update = False if needs_update: logging.debug("Updating Windows root certificates...") if os.path.isfile(cert_file): os.unlink(cert_file) from internal.os_util import run_elevated run_elevated('certutil.exe', '-generateSSTFromWU "{0}"'.format(cert_file)) if os.path.isfile(cert_file): run_elevated('certutil.exe', '-addstore -f Root "{0}"'.format(cert_file)) except Exception: pass
def requires(self, module, module_name=None): """Try importing a module and installing it if it isn't available""" ret = False if module_name is None: module_name = module try: __import__(module) ret = True except ImportError: pass if not ret: from internal.os_util import run_elevated logging.debug('Trying to install %s...', module_name) run_elevated(sys.executable, '-m pip install {0}'.format(module_name)) try: __import__(module) ret = True except ImportError: pass if not ret: print "Missing {0} module. Please run 'pip install {1}'".format(module, module_name) return ret
def find_browsers(): """Find the various known-browsers in case they are not explicitly configured""" browsers = parse_ini(os.path.join(os.path.dirname(__file__), "browsers.ini")) if browsers is None: browsers = {} plat = platform.system() if plat == "Windows": local_appdata = os.getenv('LOCALAPPDATA') program_files = str(os.getenv('ProgramFiles')) program_files_x86 = str(os.getenv('ProgramFiles(x86)')) # Allow 32-bit python to detect 64-bit browser installs if program_files == program_files_x86 and program_files.find(' (x86)') >= 0: program_files = program_files.replace(' (x86)', '') # Chrome paths = [program_files, program_files_x86, local_appdata] channels = ['Chrome', 'Chrome Beta', 'Chrome Dev'] for channel in channels: for path in paths: if path is not None and channel not in browsers: chrome_path = os.path.join(path, 'Google', channel, 'Application', 'chrome.exe') if os.path.isfile(chrome_path): browsers[channel] = {'exe': chrome_path} if local_appdata is not None and 'Canary' not in browsers: canary_path = os.path.join(local_appdata, 'Google', 'Chrome SxS', 'Application', 'chrome.exe') if os.path.isfile(canary_path): browsers['Canary'] = {'exe': canary_path} browsers['Chrome Canary'] = {'exe': canary_path} # Fall back to Chrome dev for Canary if Canary isn't available but dev is if 'Chrome Dev' in browsers and 'Canary' not in browsers: browsers['Chrome Canary'] = dict(browsers['Chrome Dev']) browsers['Canary'] = dict(browsers['Chrome Dev']) # Opera (same engine as Chrome) paths = [program_files, program_files_x86] channels = ['Opera', 'Opera beta', 'Opera developer'] for channel in channels: for path in paths: if path is not None and channel not in browsers: opera_path = os.path.join(path, channel, 'launcher.exe') if os.path.isfile(opera_path): browsers[channel] = {'exe': opera_path, 'other_exes': ['opera.exe']} # Firefox browsers paths = [program_files, program_files_x86] for path in paths: if path is not None and 'Firefox' not in browsers: firefox_path = os.path.join(path, 'Mozilla Firefox', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox' not in browsers: firefox_path = os.path.join(path, 'Firefox', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox ESR' not in browsers: firefox_path = os.path.join(path, 'Mozilla Firefox ESR', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox ESR'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox Beta' not in browsers: firefox_path = os.path.join(path, 'Mozilla Firefox Beta', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox Beta'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox Beta' not in browsers: firefox_path = os.path.join(path, 'Firefox Beta', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox Beta'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox Dev' not in browsers: firefox_path = os.path.join(path, 'Mozilla Firefox Dev', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox Dev'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox Dev' not in browsers: firefox_path = os.path.join(path, 'Firefox Dev', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox Dev'] = {'exe': firefox_path, 'type': 'Firefox'} if path is not None and 'Firefox Nightly' not in browsers: firefox_path = os.path.join(path, 'Nightly', 'firefox.exe') if os.path.isfile(firefox_path): browsers['Firefox Nightly'] = {'exe': firefox_path, 'type': 'Firefox', 'log_level': 5} # Microsoft Edge (Legacy) edge = None build = get_windows_build() if build >= 10240: edge_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'internal', 'support', 'edge', 'current', 'MicrosoftWebDriver.exe') if not os.path.isfile(edge_exe): if build > 17134: from internal.os_util import run_elevated logging.debug('Installing latest webdriver for Microsoft Edge...') run_elevated('DISM.exe', '/Online /Add-Capability ' '/CapabilityName:Microsoft.WebDriver~~~~0.0.1.0') edge_exe = os.path.join(os.environ['windir'], 'System32', 'MicrosoftWebDriver.exe') else: if build >= 17000: edge_version = 17 elif build >= 16000: edge_version = 16 elif build >= 15000: edge_version = 15 elif build >= 14000: edge_version = 14 elif build >= 10586: edge_version = 13 else: edge_version = 12 edge_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'internal', 'support', 'edge', str(edge_version), 'MicrosoftWebDriver.exe') if os.path.isfile(edge_exe): edge = {'exe': edge_exe} if edge is not None: edge['type'] = 'Edge' if 'Microsoft Edge (EdgeHTML)' not in browsers: browsers['Microsoft Edge (EdgeHTML)'] = dict(edge) if 'Microsoft Edge' not in browsers: browsers['Microsoft Edge'] = dict(edge) if 'Edge' not in browsers: browsers['Edge'] = dict(edge) # Microsoft Edge (Chromium) paths = [program_files, program_files_x86, local_appdata] channels = ['Edge Dev'] for channel in channels: for path in paths: if path is not None and channel not in browsers: edge_path = os.path.join(path, 'Microsoft', channel, 'Application', 'msedge.exe') if os.path.isfile(edge_path): browser_name = 'Microsoft {0} (Chromium)'.format(channel) if browser_name not in browsers: browsers[browser_name] = {'exe': edge_path} if local_appdata is not None and 'Microsoft Edge Canary (Chromium)' not in browsers: edge_path = os.path.join(local_appdata, 'Microsoft', 'Edge SxS', 'Application', 'msedge.exe') if os.path.isfile(edge_path): browsers['Microsoft Edge Canary (Chromium)'] = {'exe': edge_path} # Internet Explorer paths = [program_files, program_files_x86] for path in paths: if path is not None and 'IE' not in browsers: ie_path = os.path.join(path, 'Internet Explorer', 'iexplore.exe') if os.path.isfile(ie_path): browsers['ie'] = {'exe': ie_path, 'type': 'IE'} elif plat == "Linux": chrome_path = '/opt/google/chrome/chrome' if 'Chrome' not in browsers and os.path.isfile(chrome_path): browsers['Chrome'] = {'exe': chrome_path} beta_path = '/opt/google/chrome-beta/chrome' if 'Chrome Beta' not in browsers and os.path.isfile(beta_path): browsers['Chrome Beta'] = {'exe': beta_path} # google-chrome-unstable is the closest thing to Canary for Linux canary_path = '/opt/google/chrome-unstable/chrome' if os.path.isfile(canary_path): if 'Chrome Dev' not in browsers: browsers['Chrome Dev'] = {'exe': canary_path} if 'Chrome Canary' not in browsers: browsers['Chrome Canary'] = {'exe': canary_path} if 'Canary' not in browsers: browsers['Canary'] = {'exe': canary_path} # Chromium chromium_path = '/usr/lib/chromium-browser/chromium-browser' if 'Chromium' not in browsers and os.path.isfile(chromium_path): browsers['Chromium'] = {'exe': chromium_path} if 'Chrome' not in browsers and os.path.isfile(chromium_path): browsers['Chrome'] = {'exe': chromium_path} chromium_path = '/usr/bin/chromium-browser' if 'Chromium' not in browsers and os.path.isfile(chromium_path): browsers['Chromium'] = {'exe': chromium_path} if 'Chrome' not in browsers and os.path.isfile(chromium_path): browsers['Chrome'] = {'exe': chromium_path} # Opera opera_path = '/usr/lib/x86_64-linux-gnu/opera/opera' if 'Opera' not in browsers and os.path.isfile(opera_path): browsers['Opera'] = {'exe': opera_path} opera_path = '/usr/lib64/opera/opera' if 'Opera' not in browsers and os.path.isfile(opera_path): browsers['Opera'] = {'exe': opera_path} beta_path = '/usr/lib/x86_64-linux-gnu/opera-beta/opera-beta' if 'Opera beta' not in browsers and os.path.isfile(beta_path): browsers['Opera beta'] = {'exe': beta_path} beta_path = '/usr/lib64/opera-beta/opera-beta' if 'Opera beta' not in browsers and os.path.isfile(beta_path): browsers['Opera beta'] = {'exe': beta_path} dev_path = '/usr/lib/x86_64-linux-gnu/opera-developer/opera-developer' if 'Opera developer' not in browsers and os.path.isfile(dev_path): browsers['Opera developer'] = {'exe': dev_path} dev_path = '/usr/lib64/opera-developer/opera-developer' if 'Opera developer' not in browsers and os.path.isfile(dev_path): browsers['Opera developer'] = {'exe': dev_path} # Firefox browsers firefox_path = '/usr/lib/firefox/firefox' if 'Firefox' not in browsers and os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} firefox_path = '/usr/bin/firefox' if 'Firefox' not in browsers and os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} firefox_path = '/usr/lib/firefox-esr/firefox-esr' if 'Firefox' not in browsers and os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} if 'Firefox ESR' not in browsers and os.path.isfile(firefox_path): browsers['Firefox ESR'] = {'exe': firefox_path, 'type': 'Firefox'} nightly_path = '/usr/lib/firefox-trunk/firefox-trunk' if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path): browsers['Firefox Nightly'] = {'exe': nightly_path, 'type': 'Firefox', 'log_level': 5} nightly_path = '/usr/bin/firefox-trunk' if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path): browsers['Firefox Nightly'] = {'exe': nightly_path, 'type': 'Firefox', 'log_level': 5} elif plat == "Darwin": chrome_path = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' if 'Chrome' not in browsers and os.path.isfile(chrome_path): browsers['Chrome'] = {'exe': chrome_path} canary_path = '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary' if os.path.isfile(canary_path): if 'Chrome Dev' not in browsers: browsers['Chrome Dev'] = {'exe': canary_path} if 'Chrome Canary' not in browsers: browsers['Chrome Canary'] = {'exe': canary_path} if 'Canary' not in browsers: browsers['Canary'] = {'exe': canary_path} firefox_path = '/Applications/Firefox.app/Contents/MacOS/firefox' if 'Firefox' not in browsers and os.path.isfile(firefox_path): browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'} nightly_path = '/Applications/FirefoxNightly.app/Contents/MacOS/firefox' if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path): browsers['Firefox Nightly'] = {'exe': nightly_path, 'type': 'Firefox', 'log_level': 5} logging.debug('Detected Browsers:') for browser in browsers: logging.debug('%s: %s', browser, browsers[browser]['exe']) if 'Firefox' in browsers: try: # make sure marionette is up to date from internal.os_util import run_elevated run_elevated(sys.executable, '-m pip install --upgrade marionette_driver') except Exception: pass return browsers
def startup(self): """Validate that all of the external dependencies are installed""" ret = True # default /tmp/wptagent as an alive file on Linux if self.options.alive is None: if platform.system() == "Linux": self.options.alive = '/tmp/wptagent' else: self.options.alive = os.path.join(os.path.dirname(__file__), 'wptagent.alive') self.alive() ret = self.requires('dns', 'dnspython') and ret ret = self.requires('monotonic') and ret ret = self.requires('PIL', 'pillow') and ret ret = self.requires('psutil') and ret ret = self.requires('requests') and ret if not self.options.android and not self.options.iOS: ret = self.requires('tornado') and ret # Windows-specific imports if platform.system() == "Windows": ret = self.requires('win32api', 'pywin32') and ret # Optional imports self.requires('brotli') self.requires('fontTools', 'fonttools') # Try patching ws4py with a faster lib try: self.requires('wsaccel') import wsaccel wsaccel.patch_ws4py() except Exception: pass try: subprocess.check_output(['python', '--version']) except Exception: print "Make sure python 2.7 is available in the path." ret = False try: subprocess.check_output('{0} -version'.format(self.image_magick['convert']), shell=True) except Exception: print "Missing convert utility. Please install ImageMagick " \ "and make sure it is in the path." ret = False try: subprocess.check_output('{0} -version'.format(self.image_magick['mogrify']), shell=True) except Exception: print "Missing mogrify utility. Please install ImageMagick " \ "and make sure it is in the path." ret = False if platform.system() == "Linux": try: subprocess.check_output(['traceroute', '--version']) except Exception: logging.debug("Traceroute is missing, installing...") subprocess.call(['sudo', 'apt-get', '-yq', 'install', 'traceroute']) # If we are on Linux and there is no display, enable xvfb by default if platform.system() == "Linux" and not self.options.android and \ not self.options.iOS and 'DISPLAY' not in os.environ: self.options.xvfb = True if self.options.xvfb: ret = self.requires('xvfbwrapper') and ret if ret: from xvfbwrapper import Xvfb self.xvfb = Xvfb(width=1920, height=1200, colordepth=24) self.xvfb.start() # Figure out which display to capture from if platform.system() == "Linux" and 'DISPLAY' in os.environ: logging.debug('Display: %s', os.environ['DISPLAY']) self.capture_display = os.environ['DISPLAY'] elif platform.system() == "Darwin": proc = subprocess.Popen('ffmpeg -f avfoundation -list_devices true -i ""', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) _, err = proc.communicate() for line in err.splitlines(): matches = re.search(r'\[(\d+)\] Capture screen', line) if matches: self.capture_display = matches.group(1) break elif platform.system() == "Windows": self.capture_display = 'desktop' if self.options.throttle: try: subprocess.check_output('sudo cgset -h', shell=True) except Exception: print "Missing cgroups, make sure cgroup-tools is installed." ret = False # Fix Lighthouse install permissions if platform.system() != "Windows": from internal.os_util import run_elevated run_elevated('chmod', '-R 777 ~/.config/configstore/') try: import getpass run_elevated('chown', '-R {0}:{0} ~/.config'.format(getpass.getuser())) except Exception: pass # Check for Node 10+ if self.get_node_version() < 10.0: if platform.system() == "Linux": # This only works on debian-based systems logging.debug('Updating Node.js to 10.x') subprocess.call('curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -', shell=True) subprocess.call(['sudo', 'apt-get', 'install', '-y', 'nodejs']) if self.get_node_version() < 10.0: logging.warning("Node.js 10 or newer is required for Lighthouse testing") # Check the iOS install if self.ios is not None: ret = self.ios.check_install() if not self.options.android and not self.options.iOS and not self.options.noidle: self.wait_for_idle(300) if self.adb is not None: if not self.adb.start(): print "Error configuring adb. Make sure it is installed and in the path." ret = False self.shaper.remove() if not self.shaper.install(): if platform.system() == "Windows": print "Error configuring traffic shaping, make sure secure boot is disabled." else: print "Error configuring traffic shaping, make sure it is installed." ret = False # Update the Windows root certs if platform.system() == "Windows": self.update_windows_certificates() return ret
def startup(self): """Validate that all of the external dependencies are installed""" ret = True self.alive() ret = self.requires('dns', 'dnspython') and ret ret = self.requires('monotonic') and ret ret = self.requires('PIL', 'pillow') and ret ret = self.requires('psutil') and ret ret = self.requires('requests') and ret if not self.options.android and not self.options.iOS: ret = self.requires('tornado') and ret # Windows-specific imports if platform.system() == "Windows": ret = self.requires('win32api', 'pypiwin32') and ret try: subprocess.check_output(['python', '--version']) except Exception: print "Make sure python 2.7 is available in the path." ret = False try: subprocess.check_output('{0} -version'.format( self.image_magick['convert']), shell=True) except Exception: print "Missing convert utility. Please install ImageMagick " \ "and make sure it is in the path." ret = False try: subprocess.check_output('{0} -version'.format( self.image_magick['mogrify']), shell=True) except Exception: print "Missing mogrify utility. Please install ImageMagick " \ "and make sure it is in the path." ret = False if platform.system() == "Linux": try: subprocess.check_output(['traceroute', '--version']) except Exception: logging.debug("Traceroute is missing, installing...") subprocess.call( ['sudo', 'apt-get', '-yq', 'install', 'traceroute']) # if we are on Linux and there is no display, enable xvfb by default if platform.system() == "Linux" and not self.options.android and \ not self.options.iOS and 'DISPLAY' not in os.environ: self.options.xvfb = True if self.options.xvfb: ret = self.requires('xvfbwrapper') and ret if ret: from xvfbwrapper import Xvfb self.xvfb = Xvfb(width=1920, height=1200, colordepth=24) self.xvfb.start() # Figure out which display to capture from if platform.system() == "Linux" and 'DISPLAY' in os.environ: logging.debug('Display: %s', os.environ['DISPLAY']) self.capture_display = os.environ['DISPLAY'] elif platform.system() == "Darwin": proc = subprocess.Popen( 'ffmpeg -f avfoundation -list_devices true -i ""', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) _, err = proc.communicate() for line in err.splitlines(): matches = re.search(r'\[(\d+)\] Capture screen', line) if matches: self.capture_display = matches.group(1) break elif platform.system() == "Windows": self.capture_display = 'desktop' if self.options.throttle: try: subprocess.check_output('sudo cgset -h', shell=True) except Exception: print "Missing cgroups, make sure cgroup-tools is installed." ret = False # Fix Lighthouse install permissions if platform.system() != "Windows": from internal.os_util import run_elevated run_elevated('chmod', '-R 777 ~/.config/configstore/') # Check the iOS install if self.ios is not None: ret = self.ios.check_install() if not self.options.android and not self.options.iOS: self.wait_for_idle(300) if self.adb is not None: if not self.adb.start(): print "Error configuring adb. Make sure it is installed and in the path." ret = False self.shaper.remove() if not self.shaper.install(): if platform.system() == "Windows": print "Error configuring traffic shaping, make sure secure boot is disabled." else: print "Error configuring traffic shaping, make sure it is installed." ret = False return ret